package main import ( "encoding/csv" "flag" "fmt" "os" "strconv" "strings" "sync" "time" "main/httplib" "gitee.com/go-package/carbon" "github.com/cheggaaa/pb/v3" ) var RecordBaseUrl = "https://master.weimob.com/api3/ec/mbp/userrecord/pageSession" var RecordInfoUrl = "https://master.weimob.com/api3/ec/mbp/userrecord/getRecordBySession" var api = "https://dopen.weimob.com/api/2_0/ec/membership/customerRollRecord" type RecordBaseResp struct { Errcode string `json:"errcode"` Errmsg interface{} `json:"errmsg"` Data struct { TotalCount int `json:"totalCount"` PageSize int `json:"pageSize"` PageNum int `json:"pageNum"` PageList []RecordBase `json:"pageList"` } `json:"data"` GlobalTicket string `json:"globalTicket"` MonitorTrackID string `json:"monitorTrackId"` } type RecordBase struct { StartTime int64 `json:"startTime"` EndTime int64 `json:"endTime"` Pid int64 `json:"pid"` CustomerWid int64 `json:"customerWid"` StoreID int64 `json:"storeId"` KeepingTime int64 `json:"keepingTime"` Sessionid string `json:"sessionid"` StoreName string `json:"storeName"` StatType string `json:"statType"` StatTypeName string `json:"statTypeName"` InPageName string `json:"inPageName"` OutPageName string `json:"outPageName"` SharedGuide string `json:"sharedGuide"` SharedTuike string `json:"sharedTuike"` InType string `json:"inType"` KeepingTimeStr string `json:"keepingTimeStr"` } type RecordInfo struct { Pid int64 `json:"pid"` Timestamp int64 `json:"timestamp"` CustomerWid int64 `json:"customerWid"` StoreID int64 `json:"storeId"` BizType int64 `json:"bizType"` BizValue int64 `json:"bizValue"` Sessionid string `json:"sessionid"` PageName string `json:"pageName"` Content string `json:"content"` TemplateKey string `json:"templateKey"` } type RecordInfoResp struct { Errcode string `json:"errcode"` Errmsg string `json:"errmsg"` Data struct { TotalCount int64 `json:"totalCount"` Records []RecordInfo `json:"records"` } `json:"data"` GlobalTicket string `json:"globalTicket"` MonitorTrackID string `json:"monitorTrackId"` } //postData = {"pid": "100001752327","storeId": "2630133327","pageNum": 2,"pageSize": 10,"customerWid": "2636610151"} type RecordBasePostData struct { Pid int64 `json:"pid"` StoreId int64 `json:"storeId"` CustomerWid int64 `json:"customerWid"` PageNum int64 `json:"pageNum"` PageSize int64 `json:"pageSize"` Vip Vip `json:"-"` } //postData = {"pid": "100001752327","storeId": "2630133327","pageNum": 2,"pageSize": 10,"customerWid": "2636610151"} type RecordInfoPostData struct { Pid int64 `json:"pid"` StoreId int64 `json:"storeId"` CustomerWid int64 `json:"customerWid"` PageNum int64 `json:"pageNum"` PageSize int64 `json:"pageSize"` SessionId string `json:"sessionId"` } type WriteCsv struct { file *csv.Writer mu sync.Mutex } var ( brand string worker int64 start int64 token string allPids = map[string]int64{"ugg": 4019873615680, "ecco": 100001752327, "hdugg": 4019876273702} ) func NewCsv(name string) (*WriteCsv, error) { _, err := os.Stat(name) if err == nil { os.Remove(name) } f, err := os.Create(name) if err != nil { return nil, err } f.WriteString("\xEF\xBB\xBF") return &WriteCsv{file: csv.NewWriter(f), mu: sync.Mutex{}}, nil } func (w *WriteCsv) Write(data []string) { w.mu.Lock() defer w.mu.Unlock() w.file.Write(data) w.file.Flush() } func main() { flag.StringVar(&brand, "b", "", "品牌 例如 : ugg ecco") flag.StringVar(&token, "t", "", "品牌 例如 : ugg ecco") flag.Int64Var(&worker, "w", 1, "工作线程数 : 默认 1") flag.Int64Var(&start, "n", 0, "用户起始id(WID)") flag.Parse() if len(brand) == 0 { fmt.Println("需要 品牌 Like -b ugg") return } if len(token) == 0 { fmt.Println("需要 token ") return } pid, ok := allPids[brand] if !ok { fmt.Println("不存在该品牌的 PID 程序终止") return } regMysql(brand) regOrcle(brand) wids, err := GetAllWid() if err != nil { fmt.Println(err) return } if len(wids) == 0 { fmt.Println("No Data Found") return } bar := pb.StartNew(len(wids)) csvW, _ := NewCsv(brand + ".csv") csvW.Write([]string{ "Wid", "姓名", "手机", "开始时间", "结束时间", "开始时间-年", "开始时间-月", "开始时间-日", "开始时间-时", "开始时间-分", "结束时间-年", "结束时间-月", "结束时间-日", "结束时间-时", "结束时间-分", "渠道", "进入页面", "退出页", "时长", "分享导购", "分享微客", "进入方式", }) dataChan := make(chan RecordBasePostData, worker) if worker > 0 { for i := 0; i < int(worker); i++ { go RunWorker(dataChan, csvW, token) } } else { go RunWorker(dataChan, csvW, token) } for _, v := range wids { if len(v.Phone) == 0 { continue } pd := RecordBasePostData{} pd.Vip = v pd.CustomerWid = v.Wid pd.PageNum = 1 pd.PageSize = 100 pd.Pid = pid dataChan <- pd // fmt.Println(index, v.Wid, "ToDo") bar.Increment() // var wt = []string{strconv.Itoa(int(v.Wid)), v.Name, v.Phone, strconv.Itoa(len(rs.Data.PageList))} } bar.Finish() time.Sleep(10000 * time.Second) } func RunWorker(dataChan chan RecordBasePostData, wt *WriteCsv, token string) { //[Wid,姓名,手机,开始时间,结束时间,渠道,进入页面,退出页,时长,分享导购,分享微客,进入方式] for { select { case d := <-dataChan: rs, err := QueryRecordBaseOneWid(d, token) if err != nil { return } if rs.Data.TotalCount == 0 { wt.Write([]string{ strconv.Itoa(int(d.Vip.Wid)), d.Vip.Name, d.Vip.Phone, "无", "无", "无", "无", "无", "无", "无", "无", "无", "无", "无", "无", "无", "无", "无", "无", "无", "无", "无", }) } else { for _, v := range rs.Data.PageList { svs := carbon.CreateFromTimestamp(v.StartTime).Time.Format("2006-01-02-15-04") evs := carbon.CreateFromTimestamp(v.EndTime).Time.Format("2006-01-02-15-04") sv := strings.Split(svs, "-") ev := strings.Split(evs, "-") if v.InType == "" { v.InType = "无" } wt.Write([]string{ strconv.Itoa(int(d.Vip.Wid)), d.Vip.Name, d.Vip.Phone, svs, evs, sv[0], sv[1], sv[2], sv[3], sv[4], ev[0], ev[1], ev[2], ev[3], ev[4], v.StatTypeName, v.InPageName, v.OutPageName, v.KeepingTimeStr, v.SharedGuide, v.SharedTuike, v.InType, }) } } default: } } } // "cb446455d8855e88594e53588043fedfef282fed20ab59ff985888ad04b2a1c4" func QueryRecordBaseOneWid(post RecordBasePostData, token string) (*RecordBaseResp, error) { req := GetRequests(RecordBaseUrl, token) req, _ = req.JSONBody(&post) var result RecordBaseResp err := req.ToJSON(&result) if err != nil { return nil, err } return &result, nil } func QueryRecordInfoOneWid(post RecordInfoPostData, token string) (*RecordInfoResp, error) { req := GetRequests(RecordInfoUrl, token) req, _ = req.JSONBody(&post) var result RecordInfoResp err := req.ToJSON(&result) if err != nil { return nil, err } return &result, nil } func GetRequests(url string, token string) *httplib.HTTPRequest { req := httplib.Post(url) req.Header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE") req.Header("authorization", fmt.Sprintf("bearer %s", token)) return req } // func main() { // count := 100000 // // create and start new bar // bar := pb.StartNew(count) // // start bar from 'default' template // // bar := pb.Default.Start(count) // // start bar from 'simple' template // // bar := pb.Simple.Start(count) // // start bar from 'full' template // // bar := pb.Full.Start(count) // for i := 0; i < count; i++ { // bar.Increment() // time.Sleep(time.Millisecond) // } // // finish bar // bar.Finish() // }