123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037 |
- package main
- import (
- "bufio"
- "bytes"
- "flag"
- "context"
- "encoding/base64"
- "encoding/json"
- "fmt"
- "github.com/tidwall/gjson"
- "io"
- "io/ioutil"
- "unicode"
- //"log"
- //"log"
- "math/rand"
- "net/http"
- "os"
- "strconv"
- "strings"
- "time"
- "syscall"
- "os/signal"
- "path/filepath"
- "github.com/spf13/viper"
- //"github.com/chromedp/cdproto/dom"
- "github.com/xuri/excelize/v2"
- "github.com/chromedp/cdproto/network"
- "github.com/chromedp/cdproto/cdp"
- "github.com/chromedp/chromedp"
- )
- var (
- configFile = flag.String("config", "./common.yaml", "config file location")
- version = flag.Bool("version", false, "print the version")
- GitCommit = "library-import"
- Version = "library-import"
- )
- type Configure struct {
- // 处理视频
- HandleVideo bool
- // 处理简单题
- HandleAnswer bool
- // 处理网页
- HandlePage bool
- // 处理材料
- HandleMaterial bool
- // 处理不计分题
- HandelNoPoint bool
- // 处理间隔
- HandleInterval int
- // 跳过已处理
- SkipProcessed bool
- //VisitCount int
- // 提交休眠时间
- CommitExamTime int
- //SleepTime int
- // 是否手动处理
- CodeManual bool
- // 超时时间
- Timeout int
- // 图片验证token
- VerifyToken string
- VerifyType string
- // 无答案数
- QuestionCount int
- // 最大随机出错数
- MaxRandomQuestion int
- // 最小分
- MinScore int
- // 只爬取问答题
- OnlyCrawlerAnswer bool
- }
- var Conf *Configure
- var v *viper.Viper
- // LoadConfig 装载配置文件
- func LoadConfig(filename string) error {
- configPath, configName := filepath.Split(filename)
- fileList := strings.Split(configName, ".")
- if len(fileList) < 2 {
- return fmt.Errorf("%s", "文件格式不正确")
- }
- configName = fileList[0]
- fileExt := fileList[1]
- var err error
- if fileExt == "json" {
- err = LoadConfigFromJson(configName, configPath)
- } else if fileExt == "yaml" || fileExt == "yml" {
- err = LoadConfigFromYaml(configName, configPath)
- } else {
- err = fmt.Errorf("%s", "不支持的文件格式")
- }
- // 出错直接返回
- if err != nil {
- return err
- }
- return nil
- }
- // LoadConfigFromYaml 装载yaml类型的配置文件
- func LoadConfigFromYaml(configName, configPath string) error {
- v = viper.New()
- v.SetConfigName(configName)
- v.AddConfigPath(configPath)
- //设置配置文件类型
- v.SetConfigType("yaml")
- if err := v.ReadInConfig(); err != nil {
- return err
- }
- return parseConfig()
- }
- // LoadConfigFromJson 装载json类型的配置文件
- func LoadConfigFromJson(configName, configPath string) error {
- v = viper.New()
- v.SetConfigName(configName)
- v.AddConfigPath(configPath)
- //设置配置文件类型
- v.SetConfigType("json")
- if err := v.ReadInConfig(); err != nil {
- return err
- }
- return parseConfig()
- }
- func parseConfig() error {
- Conf = &Configure{}
- if err := v.Unmarshal(Conf); err != nil {
- return err
- }
- return nil
- }
- func commonVerify(image string) string {
- config := map[string]interface{}{}
- config["image"] = image
- if Conf.VerifyType == ""{
- config["type"] = "88888"
- }else{
- config["type"] = Conf.VerifyType
- }
- config["token"] = Conf.VerifyToken
- //config["token"] = "2EEYFMjDVpMTky-Z_BCgt_I14g4qq_D63S3NsQesfMc"
- configData, _ := json.Marshal(config)
- body := bytes.NewBuffer([]byte(configData))
- resp, err := http.Post("http://api.jfbym.com/api/YmServer/customApi", "application/json;charset=utf-8", body)
- defer resp.Body.Close()
- if err != nil {
- return ""
- }
- data, _ := ioutil.ReadAll(resp.Body)
- //fmt.Println(string(data), err)
- return string(data)
- }
- func codeGetNew(data []byte, account string) string {
- name := account + ".png"
- ioutil.WriteFile(name, data, 0666)
- defer func() {
- os.Remove(name)
- }()
- file, _ := os.Open(name)
- defer file.Close()
- fileInfo, _ := file.Stat()
- imageSize := fileInfo.Size()
- imageData := make([]byte, imageSize)
- _, err := file.Read(imageData)
- if err != nil {
- fmt.Println("保存文件失败")
- return ""
- }
- base64Data := base64.StdEncoding.EncodeToString(imageData)
- fmt.Println("开始识别验证码")
- respData := commonVerify(base64Data)
- //fmt.Println("验证码识别结果:", respData)
- code := gjson.Get(respData, "code").Int()
- if code != 10000 {
- fmt.Println("验证码识别失败:", gjson.Get(respData, "msg").String())
- return ""
- }
- return gjson.Get(respData, "data.data").String()
- }
- type CourseModule struct {
- Name string
- ModuleId string
- }
- func getCourseModules(courceId string, cookie string, specialCourse bool) ([]CourseModule, error) {
- url := fmt.Sprintf("https://lms.ouchn.cn/api/courses/%s/modules", courceId)
- request, err := http.NewRequest("GET", url, nil)
- if err != nil {
- return nil, err
- }
- //fmt.Println("URL :",url)
- //fmt.Println("COOKIE :",cookie)
- request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- request.Header.Set("Cookie", cookie)
- request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
- //request.Header.Set("Host", "chengdu.ouchn.cn")
- client := http.Client{}
- resp, err := client.Do(request)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
- respBytes, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return nil, err
- }
- //fmt.Println("MODEL :",string(respBytes), err)
- ret := []CourseModule{}
- mos := gjson.GetBytes(respBytes, "modules").Array()
- for _, mo := range mos {
- /*if (len(mo.Get("syllabuses").Array())) == 0 && !specialCourse {
- continue
- }*/
- item := CourseModule{}
- item.ModuleId = fmt.Sprintf("%d", mo.Get("id").Int())
- item.Name = mo.Get("name").String()
- ret = append(ret, item)
- }
- return ret, nil
- }
- type Course struct {
- Name string
- Url string
- }
- func getCourceNew(ctx context.Context) []Course {
- // https://lms.ouchn.cn/user/courses
- courseList := []Course{}
- err := chromedp.Run(ctx,
- chromedp.Navigate("https://lms.ouchn.cn/user/courses"),
- //chromedp.Sleep(2*time.Second),
- )
- if err != nil{
- fmt.Println("进入学习网失败:",err)
- }
- cookie := getCookie(ctx)
- i := int64(1)
- totalPage, err := getCourceNewImpl(&courseList,cookie,i)
- if err != nil{
- fmt.Println("获取课程失败;",err,"page=",i)
- return courseList
- }
- //fmt.Println("totalPage:",totalPage)
- if i < totalPage{
- for i=2;i<=totalPage;i++{
- _,err = getCourceNewImpl(&courseList,cookie,i )
- if err != nil{
- fmt.Println("获取课程失败;",err,"page=",i)
- continue
- //return err
- }
- }
- }
- return courseList
- }
- func getCourceNewImpl(courses *[]Course,cookie string,page int64) (int64, error) {
- url := fmt.Sprintf(`https://lms.ouchn.cn/api/my-courses?conditions={"status":["ongoing"],"keyword":""}&fields=id,name,course_code,department(id,name),grade(id,name),klass(id,name),course_type,cover,small_cover,start_date,end_date,is_started,is_closed,academic_year_id,semester_id,credit,compulsory,second_name,display_name,created_user(id,name),org(is_enterprise_or_organization),org_id,public_scope,course_attributes(teaching_class_name,copy_status,tip,data),audit_status,audit_remark,can_withdraw_course,imported_from,allow_clone,is_instructor,is_team_teaching,academic_year(id,name),semester(id,name),instructors(id,name,email,avatar_small_url),is_master,is_child,has_synchronized,master_course(name)&page=%d&page_size=10`, page)
- //fmt.Println("获取课程 url:",url)
- request, err := http.NewRequest("GET", url, nil)
- if err != nil {
- return 0, err
- }
- request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- request.Header.Set("Cookie", cookie)
- request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
- //request.Header.Set("Host", "chengdu.ouchn.cn")
- client := http.Client{}
- resp, err := client.Do(request)
- if err != nil {
- return 0, err
- }
- if resp.StatusCode != http.StatusOK {
- return 0, fmt.Errorf("wrong status code: %d", resp.StatusCode)
- }
- defer resp.Body.Close()
- respBytes, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return 0, err
- }
- //fmt.Println("RESP:",string(respBytes))
- list := gjson.GetBytes(respBytes,"courses").Array()
- for _,course := range list{
- cou := Course{}
- cou.Name = course.Get("display_name").String()
- if cou.Name == ""{
- cou.Name = course.Get("name").String()
- }
- //fmt.Println("DIS NAME :",course.Get("display_name").String(),"NAME:",course.Get("name").String())
- //fmt.Println("course:",course.String())
- cou.Url = course.Get("id").String()
- if cou.Url != ""{
- cou.Url = fmt.Sprintf("https://lms.ouchn.cn/course/%s/ng",cou.Url)
- *courses = append(*courses,cou)
- }
- }
- total_page := gjson.GetBytes(respBytes,"pages").Int()
- return total_page, nil
- }
- func getCourse(courses *[]Course,cookie string,page int64) (int64, error) {
- url := fmt.Sprintf("https://menhu.pt.ouchn.cn/ouchnapp/wap/course/xskc-pc?page=%d", page)
- fmt.Println("获取课程 :",url)
- request, err := http.NewRequest("GET", url, nil)
- if err != nil {
- return 0, err
- }
- request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- request.Header.Set("Cookie", cookie)
- request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
- //request.Header.Set("Host", "chengdu.ouchn.cn")
- client := http.Client{}
- resp, err := client.Do(request)
- if err != nil {
- return 0, err
- }
- defer resp.Body.Close()
- respBytes, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return 0, err
- }
- //fmt.Println("RESP :",string(respBytes))
- code := gjson.GetBytes(respBytes, "e").Int()
- if code == 0 {
- data := gjson.GetBytes(respBytes, "d").String()
- list := gjson.Get(data,"list").Array()
- for _,course := range list{
- cou := Course{}
- cou.Name = course.Get("name").String()
- cou.Url = course.Get("url").String()
- *courses = append(*courses,cou)
- }
- total_page := gjson.Get(data,"total_page").Int()
- return total_page, nil
- }
- return 0, fmt.Errorf("code is not 0 ")
- }
- func getCourseId(src string) string {
- array := strings.Split(src, "/")
- if len(array) < 5 {
- return ""
- }
- return array[4]
- }
- func getCookie(ctx context.Context)string {
- // 执行登录操作
- var cookies []*network.Cookie
- err := chromedp.Run(ctx,
- chromedp.ActionFunc(func(ctx context.Context) error {
- // 获取页面的 cookie
- var err error
- cookies, err = network.GetCookies().Do(ctx)
- if err != nil {
- fmt.Println("获取cookie 失败:",err)
- return err
- }
- return nil
- }),
- )
- if err != nil{
- return ""
- }
- ret := ""
- for _, v := range cookies {
- item := fmt.Sprintf("%s=%s", v.Name, v.Value)
- if ret == "" {
- ret = item
- continue
- }
- ret = ret + ";"
- ret = ret + item
- }
- return ret
- }
- type ModuleSubInfo struct {
- Name string
- Id string
- Stype string
- SyllabusId int64
- Type int // 1 page,2 vedio
- PreUnCompelte bool
- }
- type ModuleInfo struct {
- TxtInfos []ModuleSubInfo
- ExamInfos []ModuleSubInfo
- VedioInfos []ModuleSubInfo
- }
- func GetModuleInfo(courseId string, moduleId string, cookie string) (*ModuleInfo, error) {
- url := fmt.Sprintf("https://lms.ouchn.cn/api/course/%s"+
- "/all-activities?module_ids=[%s]"+
- "&activity_types=learning_activities,exams,classrooms,"+
- "live_records,rollcalls&no-loading-animation=true", courseId, moduleId)
- request, err := http.NewRequest("GET", url, nil)
- //fmt.Println("url:",url)
- if err != nil {
- return nil, err
- }
- request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- request.Header.Set("Cookie", cookie)
- request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
- //request.Header.Set("Host", "chengdu.ouchn.cn")
- client := http.Client{Timeout: 60 * time.Second}
- resp, err := client.Do(request)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
- respBytes, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return nil, err
- }
- //fmt.Println("resp:",string(respBytes))
- ret := &ModuleInfo{}
- // 试题
- exams := []ModuleSubInfo{}
- examsArray := gjson.GetBytes(respBytes, "exams").Array()
- for _, v := range examsArray {
- item := ModuleSubInfo{}
- item.Name = v.Get("title").String()
- unikey := v.Get("unique_key").String()
- array := strings.Split(unikey, "-")
- if len(array) < 2 {
- continue
- }
- item.Id = array[1]
- exams = append(exams, item)
- }
- // 文档和视频
- texts := []ModuleSubInfo{}
- vedios := []ModuleSubInfo{}
- array := gjson.GetBytes(respBytes, "learning_activities").Array()
- for _, v := range array {
- stype := v.Get("type").String()
- syllabusId := v.Get("syllabus_id").Int()
- item := ModuleSubInfo{}
- item.Name = v.Get("title").String()
- item.SyllabusId = syllabusId
- item.Stype = stype
- unikey := v.Get("unique_key").String()
- array := strings.Split(unikey, "-")
- if len(array) < 2 {
- continue
- }
- prerequisitesArray := v.Get("prerequisites").Array()
- preUnCompletion := false
- if len(prerequisitesArray) == 0 || prerequisitesArray== nil{
- preUnCompletion = true
- }else{
- for _,pre:= range prerequisitesArray{
- isPreCompltetion := pre.Get("completion_criterion.completion_key").String()
- if isPreCompltetion == "not_completed"{
- preUnCompletion = true
- }
- }
- }
- item.PreUnCompelte = preUnCompletion
- item.Id = array[1]
- if stype == "page" {
- item.Type = 1
- texts = append(texts, item)
- vedios = append(vedios, item)
- } else if stype == "online_video" || (stype == "web_link" && strings.Contains(item.Name, "视频")) || stype == "material" {
- index := -1
- item.Type = 2
- for i := len(vedios) - 1; i >= 0; i-- {
- if item.SyllabusId == vedios[i].SyllabusId {
- index = i
- break
- }
- }
- if index == -1 || index == len(vedios)-1 {
- vedios = append(vedios, item)
- } else {
- tmp := []ModuleSubInfo{}
- tmp = append(tmp, vedios[:index+1]...)
- tmp = append(tmp, item)
- tmp = append(tmp, vedios[index+1:]...)
- vedios = tmp
- }
- }
- }
- ret.ExamInfos = exams
- ret.TxtInfos = texts
- ret.VedioInfos = vedios
- return ret, nil
- }
- func getUploadIds(moduleId string, cookie string) ([]int64, error) {
- url := fmt.Sprintf("https://lms.ouchn.cn/api/activities/%s", moduleId)
- request, err := http.NewRequest("GET", url, nil)
- if err != nil {
- return nil, err
- }
- request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- request.Header.Set("Cookie", cookie)
- request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
- //request.Header.Set("Host", "chengdu.ouchn.cn")
- client := http.Client{}
- resp, err := client.Do(request)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
- respBytes, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return nil, err
- }
- ret := []int64{}
- uploads := gjson.GetBytes(respBytes, "uploads").Array()
- for _, upload := range uploads {
- id := upload.Get("id").Int()
- ret = append(ret, id)
- }
- return ret, nil
- }
- func finishUploadId(uploadId int64, moduleId string, cookie string) error {
- rBody := make(map[string]int64)
- rBody["upload_id"] = uploadId
- rByte, _ := json.Marshal(rBody)
- url := fmt.Sprintf("https://lms.ouchn.cn/api/course/activities-read/%s", moduleId)
- request, err := http.NewRequest("POST", url, bytes.NewBuffer(rByte))
- if err != nil {
- return err
- }
- request.Header.Set("Content-Type", "application/json")
- request.Header.Set("Cookie", cookie)
- request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
- //request.Header.Set("Host", "chengdu.ouchn.cn")
- client := http.Client{}
- resp, err := client.Do(request)
- if err != nil {
- return err
- }
- defer resp.Body.Close()
- respBytes, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return err
- }
- if true {
- fmt.Println("do material resp:", string(respBytes))
- }
- return nil
- }
- func finishMaterial(uploadIds []int64, moduleId string, cookie string) {
- for _, v := range uploadIds {
- finishUploadId(v, moduleId, cookie)
- time.Sleep(2 * time.Second)
- }
- }
- func doMaterial(moduleId string, cookie string) {
- uploadIds, _ := getUploadIds(moduleId, cookie)
- if len(uploadIds) > 0 {
- finishMaterial(uploadIds, moduleId, cookie)
- }
- }
- func MakeTxtUrl(courseId string, moduleId string, id string) string {
- return fmt.Sprintf("https://lms.ouchn.cn/course/%s/learning-activity/full-screen#/%s", courseId, id)
- }
- func MakeExamUrl(courseId string, moduleId string, id string) string {
- return fmt.Sprintf("https://lms.ouchn.cn/course/%s/learning-activity/full-screen#/exam/%s", courseId, id)
- }
- func VedioComplete(cookie string, id string, start, end int64) (int64, error) {
- url := fmt.Sprintf("https://lms.ouchn.cn/api/course/activities-read/%s", id)
- //fmt.Printf("get sections:%v,%v\n", str, cookie)
- m := map[string]interface{}{
- "start": start,
- "end": end,
- }
- content, _ := json.Marshal(m)
- reader := bytes.NewReader(content)
- //fmt.Printf("post data:%s\n", content)
- request, err := http.NewRequest("POST", url, reader)
- if err != nil {
- return 0, err
- }
- request.Header.Set("Content-Type", "application/json")
- request.Header.Set("Cookie", cookie)
- request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
- client := http.Client{}
- resp, err := client.Do(request)
- if err != nil {
- return 0, err
- }
- defer resp.Body.Close()
- respBytes, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return 0, err
- }
- fmt.Printf("get bytes:%s\n", respBytes)
- if gjson.GetBytes(respBytes, "error_msg").String() != ""{
- return 0,fmt.Errorf("视频处理失败(%s)",gjson.GetBytes(respBytes, "error_msg").String())
- }
- if gjson.GetBytes(respBytes, "activity_id").Int() == 0 {
- return 0, fmt.Errorf("视频处理失败")
- }
- str := gjson.GetBytes(respBytes, "data").Get("completeness").String()
- if str == "full" {
- return 100, nil
- }
- if str == "part" {
- return 10, nil
- }
- return gjson.GetBytes(respBytes, "data").Get("completeness").Int(), nil
- }
- func VedioCompleteHandle(cookie string, id string) error {
- count := 0
- start, end := int64(0), int64(1000)
- for count < 5 {
- finish, err := VedioComplete(cookie, id, start, end)
- if err != nil {
- return err
- }
- if finish < 100 {
- start = end
- end += 1000
- count++
- continue
- }
- return nil
- }
- return fmt.Errorf("视频处理失败")
- }
- func loadAnswerFromExcel() (map[string]AnswerInfo,error) {
- answerMap := make(map[string]AnswerInfo)
- entries, err := os.ReadDir("./")
- if err != nil {
- fmt.Println("Error reading directory:", err)
- return answerMap, err
- }
- for _,v := range entries{
- if !v.IsDir(){
- if strings.HasPrefix(v.Name(),"account") || strings.HasPrefix(v.Name(),"~") || strings.HasPrefix(v.Name(),".") {
- continue
- }
- if strings.HasSuffix(v.Name(),".xlsx") || strings.HasSuffix(v.Name(),".xls") {
- fmt.Println("读取文件:",v.Name())
- file, err := excelize.OpenFile(v.Name())
- if err != nil {
- fmt.Println("open file err:", err)
- return answerMap,err
- }
- for index, sheetName := range file.GetSheetList() {
- if index > 0 {
- break
- }
- rows, err := file.GetRows(sheetName)
- if err != nil{
- return answerMap,err
- }
- for _, row := range rows {
- rowLen := len(row)
- if rowLen <= 2 {
- continue
- }
- for rowIndex,_:= range row{
- row[rowIndex] = strings.TrimSpace(row[rowIndex])
- if row[rowIndex] == ""{
- fmt.Println("答案格式不正确,存在空列:",rowIndex,row)
- return answerMap,fmt.Errorf("答案格式不正确,存在空列")
- }
- }
- tmpAnswer := AnswerInfo{}
- tmpAnswer.Question = row[1]
- tmpAnswer.Question = ParseHan(tmpAnswer.Question)
- //tmpAnswer.InfoName = row[1]
- tmpAnswer.Type = row[0]
- tmpAnswer.Answer = row[2:]
- key := tmpAnswer.Type+":"+tmpAnswer.Question
- key = strings.ReplaceAll(key, " ", "")
- answerMap[key] = tmpAnswer
- }
- }
- }
- }
- //fmt.Println(v.Name(),v.IsDir())
- }
- if len(answerMap) == 0 {
- return answerMap,fmt.Errorf("答案为空")
- }
- return answerMap,nil
- }
- func handleVideo(ctx context.Context,moduleInfo *ModuleInfo, module CourseModule,courseId string,cookie string) error {
- vedioLen := len(moduleInfo.VedioInfos)
- for index, info := range moduleInfo.VedioInfos {
- if Conf.SkipProcessed{
- nextIndex := index+1
- if index+1 < vedioLen{
- if !moduleInfo.VedioInfos[nextIndex].PreUnCompelte{
- fmt.Println("已处理("+info.Name+")跳过")
- continue
- }
- }
- }
- if info.Type == 1{
- if Conf.HandlePage{
- fmt.Printf("------------开始处理网页数据:%s------------\n", info.Name)
- url := MakeTxtUrl(courseId, module.ModuleId, info.Id)
- err := chromedp.Run(ctx,
- chromedp.Navigate(url),
- )
- if err != nil{
- fmt.Println("跳转网页:",url,"失败:",err)
- return err
- }
- time.Sleep(time.Duration(Conf.HandleInterval) * time.Second)
- }
- }else if info.Type == 2 {
- if info.Stype == "material" {
- if Conf.HandleMaterial{
- fmt.Printf("------------开始处理材料数据:%s------------\n", info.Name)
- url := MakeTxtUrl(courseId, module.ModuleId, info.Id)
- err := chromedp.Run(ctx,
- chromedp.Navigate(url),
- )
- if err != nil{
- fmt.Println("跳转网页:",url,"失败:",err)
- return err
- }
- doMaterial(info.Id, cookie)
- time.Sleep(time.Duration(Conf.HandleInterval) * time.Second)
- }
- }else{
- if Conf.HandleVideo{
- fmt.Printf("------------开始处理视频数据:%s------------\n", info.Name)
- url := MakeTxtUrl(courseId, module.ModuleId, info.Id)
- err := chromedp.Run(ctx,
- chromedp.Navigate(url),
- )
- if err != nil{
- fmt.Println("跳转网页:",url,"失败:",err)
- return err
- }
- err = VedioCompleteHandle(cookie, info.Id)
- if err != nil {
- fmt.Printf("视频%s处理失败(%s)\n", info.Name, err.Error())
- return err
- }
- time.Sleep(time.Duration(Conf.HandleInterval) * time.Second)
- }
- }
- }
- }
- return nil
- }
- func handleCrawlerChooseImpl(ctx context.Context,node *cdp.Node,answerType string )(AnswerInfo,error){
- answerInfo := AnswerInfo{}
- answerInfo.Type = answerType
- var childNodes []*cdp.Node
- if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/div/span/p", &childNodes)); err != nil {
- fmt.Println("获取NODE失败:",err)
- return answerInfo,err
- }
- var question string
- if err := chromedp.Run(ctx, chromedp.Text(childNodes[0].FullXPath(), &question)); err != nil {
- fmt.Println("获取问题失败:", err)
- return answerInfo,err
- }
- question = strings.TrimSpace(question)
- fmt.Println("问题:", question)
- answerInfo.Question = question
- var childNodes1 []*cdp.Node
- if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/ol/li/label/div/span", &childNodes1)); err != nil {
- fmt.Println("获取NODE失败:",err)
- return answerInfo,err
- }
- var tmpContent string
- if err := chromedp.Run(ctx, chromedp.Text(childNodes1[0].FullXPath(), &tmpContent)); err != nil {
- fmt.Println("获取答案失败:", err)
- return answerInfo,err
- }
- if tmpContent == ""{
- if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/ol/li/label/div/span/p", &childNodes1)); err != nil {
- fmt.Println("获取NODE失败:",err)
- return answerInfo,err
- }
- }
- for _,v := range childNodes1{
- pContent := ""
- if err := chromedp.Run(ctx, chromedp.Text(v.FullXPath(), &pContent)); err != nil {
- fmt.Println("获取答案失败:", err)
- return answerInfo,err
- }
- pContent = strings.TrimSpace(pContent)
- answerInfo.Answer = append(answerInfo.Answer,pContent)
- }
- return answerInfo,nil
- }
- func handleCrawlerAnswerImpl(ctx context.Context,node *cdp.Node)(AnswerInfo,error){
- answerInfo := AnswerInfo{}
- answerInfo.Type = "问答题"
- var childNodes []*cdp.Node
- if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div[1]/div[1]/span/p[1]", &childNodes)); err != nil {
- fmt.Println("获取NODE失败:",err)
- return answerInfo ,err
- }
- var question string
- if err := chromedp.Run(ctx, chromedp.Text(childNodes[0].FullXPath(), &question)); err != nil {
- fmt.Println("获取问题失败:", err)
- return answerInfo ,err
- }
- question = strings.TrimSpace(question)
- fmt.Println("问题:", question)
- //fmt.Println(answerMap)
- answerInfo.Question = question
- return answerInfo, nil
- }
- func handleAnswerImpl(ctx context.Context,node *cdp.Node,answerMap map[string]AnswerInfo,infoName string )(int,error){
- isChoose := 1
- var childNodes []*cdp.Node
- if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div[1]/div[1]/span/p[1]", &childNodes)); err != nil {
- fmt.Println("获取NODE失败:",err)
- return isChoose ,err
- }
- var question string
- if err := chromedp.Run(ctx, chromedp.Text(childNodes[0].FullXPath(), &question)); err != nil {
- fmt.Println("获取问题失败:", err)
- return isChoose ,err
- }
- question = strings.TrimSpace(question)
- question = ParseHan(question)
- fmt.Println("问题:", question)
- infoName = strings.TrimSpace(infoName)
- key := "问答题"+":"+question
- key = strings.ReplaceAll(key, " ", "")
- tmpAnswer,ok := answerMap[key]
- if !ok{
- fmt.Println("问题:",question," 未提供答案")
- return isChoose,fmt.Errorf("无答案")
- }
- if len(tmpAnswer.Answer) == 0 {
- fmt.Println("问题:",question," 未提供答案")
- return isChoose,fmt.Errorf("无答案")
- }
- escapedAnswer := strconv.Quote(tmpAnswer.Answer[0])
- if len(tmpAnswer.Answer) > 1 {
- escapedAnswer = strconv.Quote(tmpAnswer.Answer[random(len(tmpAnswer.Answer))])
- }
- if len(escapedAnswer) == 0 {
- fmt.Println("问题:",question," 无答案")
- return isChoose,fmt.Errorf("无答案")
- }
- jsCode := fmt.Sprintf(`
- var element = document.evaluate("%s", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
- if (element) {
- element.textContent = %s;
- }
- `,node.FullXPath()+"/div/div/div/div/div/div/div/p",escapedAnswer)
- //fmt.Println("JS CODE:",jsCode)
- if err := chromedp.Run(ctx, chromedp.Evaluate(jsCode, nil)); err != nil {
- fmt.Println("设置 <p> 元素文本内容失败:", err)
- return isChoose, err
- }
- return 0, nil
- }
- func random(count int) int {
- if count == 0 {
- return 0
- }
- //rand.Seed(time.Now().UnixNano())
- return rand.Intn(count)
- }
- func handleChooseImpl(ctx context.Context,node *cdp.Node,answerMap map[string]AnswerInfo,isMulti bool,typeStr string,randomWrong bool ,infoName string)(int,error){
- miss := 1
- var childNodes []*cdp.Node
- if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/div/span/p", &childNodes)); err != nil {
- fmt.Println("获取NODE失败:",err)
- return miss ,err
- }
- var question string
- if err := chromedp.Run(ctx, chromedp.Text(childNodes[0].FullXPath(), &question)); err != nil {
- fmt.Println("获取问题失败:", err)
- return miss ,err
- }
- question = strings.TrimSpace(question)
- question = ParseHan(question)
- fmt.Println("问题:", question,"是否随机出错:",randomWrong)
- infoName = strings.TrimSpace(infoName)
- key := typeStr+":"+question
- key = strings.ReplaceAll(key, " ", "")
- fmt.Println("KEY :",key)
- tmpAnswer,ok := answerMap[key]
- if !ok{
- fmt.Println("问题:",question," 未提供答案")
- //return isChoose,nil
- }
- time.Sleep(1*time.Second)
- if isMulti{
- var nodes []*cdp.Node
- err := chromedp.Run(ctx,
- chromedp.Nodes(node.FullXPath()+"/div/div[2]/ol/li/label/span/input", &nodes, chromedp.BySearch),
- )
- if err != nil{
- fmt.Println("勾选组件不存在:",err)
- return miss,err
- }
- for index,v := range nodes{
- attr := v.AttributeValue("class")
- //fmt.Println("第",index+1," class:",attr)
- if strings.Contains(attr, "ng-not-empty") {
- fmt.Println("第",index+1,"项非空取消选择")
- if err := chromedp.Run(ctx, chromedp.Click(v.FullXPath())); err != nil {
- fmt.Println("点击取消失败:",err)
- return miss,err
- }
- }
- }
- }
- var childNodes1 []*cdp.Node
- if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/ol/li/label/div/span", &childNodes1)); err != nil {
- fmt.Println("获取NODE失败:",err)
- return miss ,err
- }
- var tmpContent string
- if err := chromedp.Run(ctx, chromedp.Text(childNodes1[0].FullXPath(), &tmpContent)); err != nil {
- fmt.Println("获取答案失败:", err)
- return miss ,err
- }
- if tmpContent == ""{
- if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/ol/li/label/div/span/p", &childNodes1)); err != nil {
- fmt.Println("获取NODE失败:",err)
- return miss ,err
- }
- }
- ///html/body/div[2]/div[4]/div[3]/div[8]/div/div/div[5]/div/div[2]/div/ol/li[18]/div/div[2]/ol/li[1]/label/div/span/p
- //fmt.Println("node:",childNodes1)
- rightIndex := -1
- for nodeIndex,v := range childNodes1{
- pContent := ""
- if err := chromedp.Run(ctx, chromedp.Text(v.FullXPath(), &pContent)); err != nil {
- fmt.Println("获取答案失败:", err)
- return miss ,err
- }
- pContent = strings.TrimSpace(pContent)
- for _,ans := range tmpAnswer.Answer{
- if ans == pContent{
- miss = 0
- fmt.Println("选择答案:", pContent)
- if err := chromedp.Run(ctx, chromedp.Click(v.FullXPath())); err != nil {
- fmt.Println("点击失败:",err)
- return miss, err
- }
- rightIndex = nodeIndex
- }
- }
- }
- if miss == 1 {
- index := random(len(childNodes1))
- fmt.Println("未找到答案,随机选择第",index+1,"项")
- if err := chromedp.Run(ctx, chromedp.Click(childNodes1[index].FullXPath())); err != nil {
- fmt.Println("点击失败:",err)
- return miss,err
- }
- }else{
- if randomWrong{
- index := 0
- for i:=0;i<10;i++ {
- index = random(len(childNodes1))
- if rightIndex == index{
- continue
- }else{
- break
- }
- }
- fmt.Println("随机出错,随机选择第",index+1,"项")
- if err := chromedp.Run(ctx, chromedp.Click(childNodes1[index].FullXPath())); err != nil {
- fmt.Println("点击失败:",err)
- return miss,err
- }
- }
- }
- return miss, nil
- }
- type AnswerInfo struct{
- Question string
- Type string
- InfoName string
- Answer []string
- }
- func checkExamFinish(id string, cookie string) (finish bool, submited bool, noScore bool) {
- url := fmt.Sprintf("https://lms.ouchn.cn/api/exams/%s", id)
- request, err := http.NewRequest("GET", url, nil)
- if err != nil {
- return false, false, false
- }
- request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- request.Header.Set("Cookie", cookie)
- request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
- //request.Header.Set("Host", "chengdu.ouchn.cn")
- client := http.Client{}
- resp, err := client.Do(request)
- if err != nil {
- return false, false, false
- }
- defer resp.Body.Close()
- respBytes, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return false, false, false
- }
- scorePercentage := gjson.GetBytes(respBytes, "score_percentage").String()
- if scorePercentage == "0" || scorePercentage == "0.0" || scorePercentage == "0.00" {
- return true, false, true
- }
- submitTimes := gjson.GetBytes(respBytes, "submit_times").Int()
- submittedTimes := gjson.GetBytes(respBytes, "submitted_times").Int()
- if submittedTimes > 0 {
- submited = true
- }
- if submitTimes == submittedTimes && submitTimes > 0 {
- return true, submited, false
- }
- return false, submited, false
- }
- func checkExamScore(id string, cookie string) (finish bool) {
- url := fmt.Sprintf("https://lms.ouchn.cn/api/exams/%s/submissions", id)
- request, err := http.NewRequest("GET", url, nil)
- if err != nil {
- return false
- }
- request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- request.Header.Set("Cookie", cookie)
- request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
- //request.Header.Set("Host", "chengdu.ouchn.cn")
- client := http.Client{}
- resp, err := client.Do(request)
- if err != nil {
- return false
- }
- defer resp.Body.Close()
- respBytes, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return false
- }
- examScore := gjson.GetBytes(respBytes, "exam_score").Int()
- fmt.Println("分数:", examScore, " 最低要求分数:", Conf.MinScore)
- if int(examScore) >= Conf.MinScore {
- return true
- }
- return false
- }
- func writeExameToExcel(infos []AnswerInfo,writer *excelize.StreamWriter,lastLine int,infoName string)(int,error){
- for _,v := range infos{
- lastLine++
- cell, err := excelize.CoordinatesToCellName(1,lastLine)
- if err != nil{
- return lastLine,err
- }
- rowList := []interface{}{}
- rowList = append(rowList,v.Type)
- rowList = append(rowList,v.Question)
- //rowList = append(rowList,infoName)
- for _,ans := range v.Answer{
- rowList = append(rowList,ans)
- }
- err = writer.SetRow(cell, rowList)
- if err != nil{
- return lastLine,err
- }
- }
- return lastLine,nil
- }
- func createFile(fileName string) (*excelize.File, *excelize.StreamWriter) {
- file := excelize.NewFile()
- //设置表名
- file.SetSheetName("Sheet1", "表1")
- //创建流式写入
- writer, err := file.NewStreamWriter("表1")
- if err != nil {
- return nil, nil
- }
- file.SaveAs(fileName)
- return file, writer
- }
- func pullBottom(ctx context.Context,needPull bool ) error {
- if needPull{
- err := waitVisibleTimeout(ctx,"/html/body/div/div/div/div/div[2]/div[2]/div[1]")
- if err != nil {
- fmt.Println("获取div失败:",err)
- return err
- }
- if err := chromedp.Run(ctx,
- chromedp.Evaluate(`document.evaluate("/html/body/div/div/div/div/div[2]/div[2]/div[1]", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.scrollTop = document.evaluate("/html/body/div/div/div/div/div[2]/div[2]/div[1]", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.scrollHeight`, nil),
- ); err != nil {
- fmt.Println("拉动窗口失败:",err )
- return err
- }
- }
- err := waitVisibleAndClickTimeout(ctx,"//a[@class='button button-green take-exam ng-scope']")
- if err != nil {
- fmt.Println("开始答题失败,找不到开始按钮:",err)
- }
- err = waitVisibleAndClickTimeout(ctx,"//input[@type='checkbox' and @name='confirm']")
- if err != nil {
- fmt.Println("开始答题失败,找不到确认按钮:",err)
- return err
- }
- return nil
- }
- // 爬取试题
- func crawlerExame(ctx context.Context,moduleInfo *ModuleInfo, module CourseModule,courseId string,writer *excelize.StreamWriter,lastLine int) (int,error) {
- for _, info := range moduleInfo.ExamInfos {
- info.Name = strings.TrimSpace(info.Name)
- url := MakeExamUrl(courseId, module.ModuleId, info.Id)
- err := chromedp.Run(ctx,
- chromedp.Navigate(url),
- )
- if err != nil{
- return lastLine,err
- }
- fmt.Println("INFO NAME:",info.Name )
- needPull := false
- for i:=0 ;i<=3;i++{
- err = pullBottom(ctx,needPull)
- if err != nil{
- needPull = true
- time.Sleep(3*time.Second)
- fmt.Println("第",i+1,"次失败:",err)
- continue
- }else{
- break
- }
- }
- if err != nil {
- fmt.Println("开始答题失败:",err)
- return lastLine,err
- }
- time.Sleep(1*time.Second)
- err = waitVisibleAndClickTimeout(ctx,"//button[text()='开始答题' or text()='继续答题']")
- if err != nil {
- fmt.Println("开始答题失败:",err)
- return lastLine,err
- }
- err = waitVisibleTimeout(ctx,"//li[contains(@class, 'subject ng-scope ')]")
- if err != nil {
- fmt.Println("等待题目出错:",err)
- return lastLine,err
- }
- var questionClasses []*cdp.Node
- if err := chromedp.Run(ctx, chromedp.Nodes("//li[contains(@class, 'subject ng-scope ')]", &questionClasses)); err != nil {
- fmt.Println("获取题目出错:",err)
- return lastLine,err
- }
- answerInfoList := []AnswerInfo{}
- fmt.Println(questionClasses,"------------------题目长度(包含题目类型):",len(questionClasses))
- for _, v := range questionClasses {
- attr := v.AttributeValue("class")
- if strings.Contains(attr, "true_or_false") {
- fmt.Println("判断题")
- answerInfo ,err := handleCrawlerChooseImpl(ctx,v,"判断题")
- if err != nil{
- return lastLine,err
- }
- answerInfoList = append(answerInfoList,answerInfo)
- }else if strings.Contains(attr, "single_selection") {
- fmt.Println("单选题")
- answerInfo ,err := handleCrawlerChooseImpl(ctx,v,"单选题")
- if err != nil{
- return lastLine,err
- }
- answerInfoList = append(answerInfoList,answerInfo)
- }else if strings.Contains(attr, "multiple_selection") {
- fmt.Println("多选题")
- answerInfo ,err := handleCrawlerChooseImpl(ctx,v,"多选题")
- if err != nil{
- return lastLine,err
- }
- answerInfoList = append(answerInfoList,answerInfo)
- } else if strings.Contains(attr, "short_answer") {
- fmt.Println("问答题")
- answerInfo ,err := handleCrawlerAnswerImpl(ctx,v)
- if err != nil{
- return lastLine,err
- }
- answerInfoList = append(answerInfoList,answerInfo)
- }else{
- fmt.Println("非题目")
- continue
- }
- }
- lastLine, err = writeExameToExcel(answerInfoList,writer,lastLine,info.Name)
- if err != nil{
- return lastLine,err
- }
- time.Sleep(time.Duration(Conf.HandleInterval)*time.Second)
- }
- return lastLine,nil
- }
- func ParseHan(str string) string {
- ret := ""
- for _, r := range str {
- if unicode.Is(unicode.Han, r) {
- ret = ret + string(r)
- } else if (r >= 48 && r <= 57) || (r >= 65 && r <= 90) || (r >= 97 && r <= 112) {
- ret = ret + string(r)
- }
- }
- return ret
- }
- func handleExame(ctx context.Context,moduleInfo *ModuleInfo, module CourseModule,courseId string,cookie string,answerMap map[string]AnswerInfo) error {
- for _, info := range moduleInfo.ExamInfos {
- info.Name = strings.TrimSpace(info.Name)
- fmt.Println("------------开始处理试题:",info.Name,"------------")
- finsh, submited, noScore := checkExamFinish(info.Id, cookie)
- if !Conf.HandelNoPoint{
- if noScore {
- fmt.Printf("不计分,该试题不做\n")
- continue
- }
- }
- if finsh {
- fmt.Printf("试题达到最大提交次数\n")
- continue
- }
- if submited {
- isFinsh := checkExamScore(info.Id, cookie)
- if isFinsh {
- fmt.Printf("已完成该试题,并满足分数\n")
- continue
- } else {
- fmt.Printf("已完成该试题,不满足分数,继续答题\n")
- }
- }
- url := MakeExamUrl(courseId, module.ModuleId, info.Id)
- err := chromedp.Run(ctx,
- chromedp.Navigate(url),
- )
- if err != nil{
- return err
- }
- needPull := false
- for i:=0 ;i<=3;i++{
- err = pullBottom(ctx,needPull)
- if err != nil{
- needPull = true
- time.Sleep(3*time.Second)
- fmt.Println("第",i+1,"次失败:",err)
- continue
- }else{
- break
- }
- }
- if err != nil {
- fmt.Println("开始答题失败:",err)
- return err
- }
- time.Sleep(1*time.Second)
- err = waitVisibleAndClickTimeout(ctx,"//button[text()='开始答题' or text()='继续答题']")
- if err != nil {
- fmt.Println("开始答题失败:",err)
- return err
- }
- err = waitVisibleTimeout(ctx,"//li[contains(@class, 'subject ng-scope ')]")
- if err != nil {
- fmt.Println("等待题目出错:",err)
- return err
- }
- time.Sleep(3*time.Second)
- //time.Sleep(10*time.Second)
- var questionClasses []*cdp.Node
- if err := chromedp.Run(ctx, chromedp.Nodes("//li[contains(@class, 'subject ng-scope ')]", &questionClasses)); err != nil {
- fmt.Println("获取题目出错:",err)
- return err
- }
- total := 0
- miss := 0
- randomCount := random(Conf.MaxRandomQuestion)
- randomHandelCount := 0
- fmt.Println("可出错个数:",randomCount,Conf.MaxRandomQuestion)
- fmt.Println("------------------题目长度(包含题目类型):",len(questionClasses))
- for _, v := range questionClasses {
- //fmt.Println("题目")
- randWrong := false
- if randomCount > randomHandelCount{
- isWrong := random(2)
- //fmt.Println("isWrong:",isWrong)
- if isWrong == 1{
- randomHandelCount++
- fmt.Println("已随机个数:",randomHandelCount)
- randWrong = true
- //fmt.Println("randWrong111111111111111",randWrong)
- }
- }
- //fmt.Println("randWrong",randWrong)
- attr := v.AttributeValue("class")
- if strings.Contains(attr, "true_or_false") {
- //fmt.Println("判断题")
- isChoose ,err := handleChooseImpl(ctx,v,answerMap,false,"判断题",randWrong,info.Name)
- if err != nil{
- return err
- }
- miss = miss + isChoose
- }else if strings.Contains(attr, "single_selection") {
- //fmt.Println("单选题")
- isChoose ,err := handleChooseImpl(ctx,v,answerMap,false,"单选题",randWrong,info.Name)
- if err != nil{
- return err
- }
- miss = miss + isChoose
- }else if strings.Contains(attr, "multiple_selection") {
- //fmt.Println("多选题")
- isChoose ,err := handleChooseImpl(ctx,v,answerMap,true,"多选题",randWrong,info.Name)
- if err != nil{
- return err
- }
- miss = miss + isChoose
- } else if strings.Contains(attr, "short_answer") {
- //fmt.Println("问答题")
- isChoose ,err := handleAnswerImpl(ctx,v,answerMap,info.Name)
- if err != nil{
- return err
- }
- miss = miss + isChoose
- }else{
- if randWrong {
- randomHandelCount--
- }
- //fmt.Println("非题目")
- continue
- }
- total++
- if miss > Conf.QuestionCount{
- fmt.Println("无答案数(",miss,")超过设定值:",Conf.QuestionCount)
- return fmt.Errorf("无答案数过多超过设定值")
- }
- }
- if miss > Conf.QuestionCount{
- fmt.Println("无答案数(",miss,")超过设定值:",Conf.QuestionCount)
- return fmt.Errorf("无答案数过多超过设定值")
- }
- fmt.Println("等待:",Conf.CommitExamTime,"秒提交")
- time.Sleep(time.Duration(Conf.CommitExamTime)*time.Second)
- err = waitVisibleAndClickTimeout(ctx,"//a[@class='button button-green ng-scope' and text()='交卷']")
- if err != nil {
- fmt.Println("未找到交卷按钮:",err)
- return err
- }
- time.Sleep(2*time.Second)
- nodes ,err := getNodeTimeout(ctx,"#submit-exam-confirmation-popup > div > div:nth-child(3) > div > button:nth-child(1)")
- if err != nil{
- fmt.Println("获取确认节点失败")
- nodes ,err = getNodeTimeout(ctx,`*[@id="submit-exam-confirmation-popup"]/div/div[3]/div/button[1]`)
- if err != nil{
- fmt.Println("获取确认节点失败")
- nodes ,err = getNodeTimeout(ctx,`/html[1]/body[1]/div[10]/div[1]/div[3]/div[1]/button[1]`)
- if err != nil{
- fmt.Println("获取确认节点失败")
- return err
- }
- }
- }
- for _, node := range nodes {
- if node.FullXPath() == `/html[1]/body[1]/div[10]/div[1]/div[3]/div[1]/button[1]`{
- fmt.Println("确定提交")
- //time.Sleep(100*time.Hour)
- err = chromedp.Run(ctx,
- chromedp.Click(node.FullXPath(), chromedp.BySearch),
- )
- if err != nil{
- fmt.Println("提交答案失败:",err)
- }
- }
- }
- fmt.Println("等待时间间隔(handleInterval):",Conf.HandleInterval)
- time.Sleep(time.Duration(Conf.HandleInterval)*time.Second)
- }
- return nil
- }
- func waitVisibleTimeout(ctx context.Context,sel interface{}) error {
- ctxTmp, cancleTmp := context.WithTimeout(ctx, time.Duration(Conf.Timeout)*time.Second)
- defer cancleTmp()
- done := make(chan struct{})
- go func() {
- defer close(done)
- err := chromedp.Run(ctxTmp,
- //chromedp.Sleep(1*time.Second),
- chromedp.WaitVisible(sel),
- )
- if err != nil{
- fmt.Println("操作:",sel,"失败 error:",err)
- //fmt.Println(sel," err:",err)
- }
- }()
- select {
- case <-done:
- // 操作完成,不需要做任何事情
- return nil
- case <-ctxTmp.Done():
- // 如果 ctxTmp 超时或取消,输出相应信息
- err := ctxTmp.Err()
- if err != nil{
- fmt.Println("操作超时,取消操作")
- return err
- }
- }
- return nil
- }
- func waitVisibleAndClickTimeout(ctx context.Context,sel interface{}) error {
- ctxTmp, cancleTmp := context.WithTimeout(ctx, time.Duration(Conf.Timeout)*time.Second)
- defer cancleTmp()
- done := make(chan struct{})
- go func() {
- defer close(done)
- err := chromedp.Run(ctxTmp,
- //chromedp.Sleep(1*time.Second),
- chromedp.WaitVisible(sel),
- chromedp.Click(sel),
- )
- if err != nil{
- fmt.Println("操作:",sel,"失败 error:",err)
- //fmt.Println(sel," err:",err)
- }
- }()
- select {
- case <-done:
- // 操作完成,不需要做任何事情
- return nil
- case <-ctxTmp.Done():
- // 如果 ctxTmp 超时或取消,输出相应信息
- err := ctxTmp.Err()
- if err != nil{
- fmt.Println("操作超时,取消操作")
- return err
- }
- }
- return nil
- }
- func getNodeTimeout(ctx context.Context,sel interface{}) ([]*cdp.Node,error) {
- var nodes []*cdp.Node
- ctxTmp, cancleTmp := context.WithTimeout(ctx, time.Duration(Conf.Timeout)*time.Second)
- defer cancleTmp()
- done := make(chan struct{})
- go func() {
- defer close(done)
- err := chromedp.Run(ctxTmp,
- chromedp.Nodes(sel, &nodes),
- )
- if err != nil{
- fmt.Println("操作:",sel,"失败 error:",err)
- //fmt.Println(sel," err:",err)
- }
- }()
- select {
- case <-done:
- // 操作完成,不需要做任何事情
- return nodes,nil
- case <-ctxTmp.Done():
- // 如果 ctxTmp 超时或取消,输出相应信息
- err := ctxTmp.Err()
- if err != nil{
- fmt.Println("操作超时,取消操作")
- return nodes,err
- }
- }
- return nodes,nil
- }
- func autoLogin(ctx context.Context,userName string) error {
- var buf []byte
- err := chromedp.Run(ctx,
- chromedp.Screenshot("body", &buf,chromedp.NodeVisible, chromedp.ByQuery),
- )
- if err != nil{
- fmt.Println("截图失败:",err)
- return err
- }
- code := codeGetNew(buf,userName)
- //fmt.Println("识别验证码位置:", code)
- codeList := strings.Split(code, "|")
- if len(codeList) <=2{
- return fmt.Errorf("识别异常,小于等于两个验证码")
- }
- fmt.Println("识别成功")
- for _, v := range codeList {
- codeX := strings.Split(v, ",")
- if len(codeX) < 2 {
- return fmt.Errorf("识别异常")
- }
- x, _ := strconv.Atoi(codeX[0])
- y, _ := strconv.Atoi(codeX[1])
- //fmt.Println("点击:",float64(x),float64(y))
- err = chromedp.Run(ctx, chromedp.MouseClickXY(float64(x),float64(y),chromedp.ButtonLeft))
- if err != nil {
- fmt.Println("点击错误:", err)
- return err
- }
- time.Sleep(1 * time.Second)
- }
- // 执行登录操作
- //fmt.Println("点击确定")
- err = chromedp.Run(ctx,
- chromedp.Click(`/html/body/div[4]/div[1]/div[1]/div[2]/div/div/div[2]`),
- )
- if err != nil{
- fmt.Println("点击确定按钮失败:",err )
- return err
- }
- return nil
- }
- func run(name,userName ,password string,answerMap map[string]AnswerInfo,handleCourse string) error {
- userName = strings.TrimSpace(userName)
- password = strings.TrimSpace(password)
- // 设置Chrome选项
- opts := []chromedp.ExecAllocatorOption{
- chromedp.Flag("disable-blink-features", "AutomationControlled"),
- chromedp.Flag("exclude-switches", "enable-automation"),
- chromedp.Flag("disable-popup-blocking", true),
- chromedp.Flag("start-maximized", true),
- chromedp.Flag("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"),
- }
- // 初始化Chrome实例
- allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...)
- defer allocCancel()
- var err error
- ctx, cancel := chromedp.NewContext(allocCtx)
- defer cancel()
- go func() {
- <-ctx.Done()
- err := ctx.Err()
- if err == context.Canceled {
- fmt.Println("浏览器已手动关闭")
- } else {
- fmt.Println("上下文发生错误:", err)
- }
- }()
- // 执行登录操作
- //fmt.Println(name,"登录")
- err = chromedp.Run(ctx,
- chromedp.Navigate("https://iam.pt.ouchn.cn/am/UI/Login?realm=%2F&service=initService&goto=https%3A%2F%2Fiam.pt.ouchn.cn%2Fam%2Foauth2%2Fauthorize%3Fservice%3DinitService%26response_type%3Dcode%26client_id%3D345fcbaf076a4f8a%26scope%3Dall%26redirect_uri%3Dhttps%253A%252F%252Fmenhu.pt.ouchn.cn%252Fouchnapp%252Fwap%252Flogin%252Findex%26decision%3DAllow"),
- chromedp.SetValue(`//*[@id="loginName"]`, userName),
- chromedp.SetValue(`//*[@id="password"]`, password),
- chromedp.Sleep(2*time.Second),
- chromedp.Click(`//*[@id="form_button"]`),
- chromedp.Sleep(1*time.Second),
- )
- if err != nil{
- fmt.Println("登录失败:",err)
- return err
- }
- fmt.Println("获取验证码")
- err = waitVisibleTimeout(ctx,"/html/body/div[4]/div[1]/div[1]/div[2]/div")
- if err != nil{
- fmt.Println("获取验证码登录失败:",err)
- return err
- }
- time.Sleep(1*time.Second)
- if !Conf.CodeManual{
- for count := 0;count < 3 ;count++{
- err = autoLogin(ctx,userName)
- if err == nil {
- break
- }else{
- time.Sleep(10*time.Second)
- }
- }
- }
- fmt.Printf("等待进入学生主页\n")
- err = waitVisibleTimeout(ctx,"//a[text()='个人信息']")
- if err != nil{
- fmt.Println("进入学生主页失败:",err)
- return err
- }
- cookie := getCookie(ctx)
- time.Sleep(1*time.Second)
- // 获取课程列表
- courseList := []Course{}
- for i:=0 ;i<3 ;i++{
- fmt.Println("开始获取课程列表")
- courseList = getCourceNew(ctx)
- if len(courseList) == 0 {
- fmt.Println("获取课程失败,课程为空,重新获取")
- time.Sleep(10*time.Second)
- continue
- }else{
- break
- }
- }
- if len(courseList) == 0 {
- fmt.Println("获取课程失败,课程为空")
- return fmt.Errorf("获取课程失败,课程为空")
- }
- for _,v := range courseList{
- if !strings.Contains(handleCourse,v.Name){
- continue
- }
- fmt.Printf("*************正在处理课程(%s)***************\n",v.Name)
- err = chromedp.Run(ctx,
- chromedp.Navigate(v.Url),
- //chromedp.WaitVisible(`//*[@id="student-module-menu"]/div[1]/div[1]/div/a`),
- )
- if err != nil {
- fmt.Println("进入课程失败:",err)
- return err
- }
- err = waitVisibleTimeout(ctx,`//*[@id="student-module-menu"]/div[1]/div[1]/div/a`)
- if err != nil {
- fmt.Println("进入课程失败:",err)
- return err
- }
- courseId := getCourseId(v.Url)
- if courseId == "" {
- fmt.Println("课程id获取失败")
- return fmt.Errorf("课程id获取失败")
- }
- cookie = getCookie(ctx)
- modules, err := getCourseModules(courseId,cookie , false)
- if err != nil {
- fmt.Printf("获取module信息失败:%s\n", err.Error())
- return err
- }
- lastLine := 0
- var writer *excelize.StreamWriter
- var file *excelize.File
- if Conf.OnlyCrawlerAnswer{
- fmt.Printf("*************正在爬取课程(%s)***************\n",v.Name)
- file ,writer = createFile(fmt.Sprintf("%s.xlsx",v.Name))
- }
- for _, module := range modules {
- fmt.Printf("***********课程:%s 栏目:%s***********\n", v.Name, module.Name)
- moduleInfo, err := GetModuleInfo(courseId, module.ModuleId, cookie)
- if err != nil {
- fmt.Printf("获取module(%s)详细信息失败:%s", module.Name, err.Error())
- return err
- }
- time.Sleep(2*time.Second)
- if Conf.OnlyCrawlerAnswer{
- lastLine, err = crawlerExame(ctx,moduleInfo,module,courseId,writer,lastLine)
- if err != nil{
- return err
- }
- file.Save()
- continue
- }
- if Conf.HandleVideo {
- err = handleVideo(ctx,moduleInfo,module,courseId,cookie)
- if err != nil{
- return err
- }
- }
- if Conf.HandleAnswer{
- err = handleExame(ctx,moduleInfo,module,courseId,cookie,answerMap)
- if err != nil{
- return err
- }
- }
- }
- //time.Sleep(100*time.Hour)
- }
- fmt.Println("学生(",name,")处理完成")
- time.Sleep(2*time.Second)
- return nil
- }
- func GetRecord(filename string) map[string]bool {
- ret := map[string]bool{}
- fi, err := os.Open(filename)
- if err != nil {
- return ret
- }
- defer fi.Close()
- br := bufio.NewReader(fi)
- for {
- a, _, c := br.ReadLine()
- //fmt.Printf("xx:%s\n", a)
- if c == io.EOF {
- break
- }
- array := strings.Split(string(a), " ")
- if len(array) != 3 {
- continue
- }
- ret[array[0]] = true
- }
- return ret
- }
- func GetAcounts(filename string) ([][]string, error) {
- visited := GetRecord("record.txt")
- fi, err := os.Open(filename)
- if err != nil {
- return nil, err
- }
- defer fi.Close()
- ret := [][]string{}
- br := bufio.NewReader(fi)
- for {
- line, err := br.ReadString('\n')
- /*a, _, c := br.ReadLine()
- if c == io.EOF {
- break
- }*/
- //line := string(a)
- //line = FilterANSI(line)
- if err != nil{
- break
- }
- array := strings.Split(line, ";")
- fmt.Println(line)
- if len(array) != 3 {
- continue
- }
- if visited[array[0]] {
- //fmt.Printf("visited:%v\n", array[2])
- continue
- }
- item := []string{array[0], array[1], array[2]}
- fmt.Println(array)
- ret = append(ret, item)
- }
- return ret, nil
- }
- func columnIndexToName(index int) string {
- var name string
- for index > 0 {
- index--
- name = string('A'+index%26) + name
- index /= 26
- }
- return name
- }
- func saveCell(file *excelize.File ,sheetName string,index ,line int,value string) error {
- cell := columnIndexToName(index) + strconv.Itoa(line) // 新列的单元格位置
- //fmt.Println("CELL VALUE :",cell,value,index,line)
- if err := file.SetCellValue(sheetName, cell,value); err != nil {
- fmt.Println(err)
- return err
- }else{
- file.SetRowHeight(sheetName, line,13.5)
- //file.SetColWidth(sheetName, columnIndexToName(index), columnIndexToName(index), 11.93) // 设置 A 列的宽度为 30
- err = file.Save()
- if err != nil{
- return err
- }
- }
- return nil
- }
- func main(){
- err := LoadConfig(*configFile)
- if err != nil {
- fmt.Println("读取配置文件出错:",err)
- return
- }
- var answerMap map[string]AnswerInfo
- if Conf.HandleAnswer {
- fmt.Println("开始收集答案")
- answerMap ,err = loadAnswerFromExcel()
- if err != nil{
- fmt.Println("收集答案出错:",err)
- return
- }
- }
- //fmt.Println("答案:",answerMap)
- go func(){
- accountFile, err := excelize.OpenFile("account.xlsx")
- if err != nil {
- fmt.Println("open file err:", err)
- return
- }
- for index, sheetName := range accountFile.GetSheetList() {
- if index > 0 {
- break
- }
- rows, err := accountFile.GetRows(sheetName)
- if err != nil {
- return
- }
- for rowLine, row := range rows {
- fmt.Println("开始处理学生:",row[2])
- rowLen := len(row)
- if rowLen < 4 {
- continue
- }
- if rowLen > 4{
- if row[4] == "已处理"{
- fmt.Println("已完成处理学生:",row[2])
- continue
- }
- }
- err = run(row[2],row[0],row[1],answerMap,row[3])
- if err != nil{
- fmt.Println("处理学生:",row[2],"出错:",err)
- }else{
- saveCell(accountFile,sheetName,5,rowLine+1,"已处理")
- }
- }
- }
- os.Exit(0)
- }()
- // 捕获信号
- sigChan := make(chan os.Signal, 1)
- signal.Notify(sigChan, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)
- sigValue := <-sigChan
- fmt.Printf("接收到退出信号:%v", sigValue)
- }
|