|
@@ -0,0 +1,2037 @@
|
|
|
|
|
+package main
|
|
|
|
|
+
|
|
|
|
|
+import (
|
|
|
|
|
+ "bufio"
|
|
|
|
|
+ "bytes"
|
|
|
|
|
+ "flag"
|
|
|
|
|
+ "context"
|
|
|
|
|
+ "encoding/base64"
|
|
|
|
|
+ "encoding/json"
|
|
|
|
|
+ "fmt"
|
|
|
|
|
+ "github.com/tidwall/gjson"
|
|
|
|
|
+ "io"
|
|
|
|
|
+ "io/ioutil"
|
|
|
|
|
+ "unicode"
|
|
|
|
|
+
|
|
|
|
|
+ //"log"
|
|
|
|
|
+
|
|
|
|
|
+ //"log"
|
|
|
|
|
+ "math/rand"
|
|
|
|
|
+ "net/http"
|
|
|
|
|
+ "os"
|
|
|
|
|
+ "strconv"
|
|
|
|
|
+ "strings"
|
|
|
|
|
+ "time"
|
|
|
|
|
+ "syscall"
|
|
|
|
|
+ "os/signal"
|
|
|
|
|
+ "path/filepath"
|
|
|
|
|
+ "github.com/spf13/viper"
|
|
|
|
|
+ //"github.com/chromedp/cdproto/dom"
|
|
|
|
|
+ "github.com/xuri/excelize/v2"
|
|
|
|
|
+ "github.com/chromedp/cdproto/network"
|
|
|
|
|
+ "github.com/chromedp/cdproto/cdp"
|
|
|
|
|
+ "github.com/chromedp/chromedp"
|
|
|
|
|
+)
|
|
|
|
|
+var (
|
|
|
|
|
+ configFile = flag.String("config", "./common.yaml", "config file location")
|
|
|
|
|
+ version = flag.Bool("version", false, "print the version")
|
|
|
|
|
+ GitCommit = "library-import"
|
|
|
|
|
+ Version = "library-import"
|
|
|
|
|
+)
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+type Configure struct {
|
|
|
|
|
+ // 处理视频
|
|
|
|
|
+ HandleVideo bool
|
|
|
|
|
+ // 处理简单题
|
|
|
|
|
+ HandleAnswer bool
|
|
|
|
|
+ // 处理网页
|
|
|
|
|
+ HandlePage bool
|
|
|
|
|
+ // 处理材料
|
|
|
|
|
+ HandleMaterial bool
|
|
|
|
|
+ // 处理不计分题
|
|
|
|
|
+ HandelNoPoint bool
|
|
|
|
|
+ // 处理间隔
|
|
|
|
|
+ HandleInterval int
|
|
|
|
|
+ // 跳过已处理
|
|
|
|
|
+ SkipProcessed bool
|
|
|
|
|
+ //VisitCount int
|
|
|
|
|
+ // 提交休眠时间
|
|
|
|
|
+ CommitExamTime int
|
|
|
|
|
+ //SleepTime int
|
|
|
|
|
+ // 是否手动处理
|
|
|
|
|
+ CodeManual bool
|
|
|
|
|
+ // 超时时间
|
|
|
|
|
+ Timeout int
|
|
|
|
|
+ // 图片验证token
|
|
|
|
|
+ VerifyToken string
|
|
|
|
|
+ VerifyType string
|
|
|
|
|
+ // 无答案数
|
|
|
|
|
+ QuestionCount int
|
|
|
|
|
+ // 最大随机出错数
|
|
|
|
|
+ MaxRandomQuestion int
|
|
|
|
|
+ // 最小分
|
|
|
|
|
+ MinScore int
|
|
|
|
|
+ // 只爬取问答题
|
|
|
|
|
+ OnlyCrawlerAnswer bool
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+var Conf *Configure
|
|
|
|
|
+var v *viper.Viper
|
|
|
|
|
+
|
|
|
|
|
+// LoadConfig 装载配置文件
|
|
|
|
|
+func LoadConfig(filename string) error {
|
|
|
|
|
+ configPath, configName := filepath.Split(filename)
|
|
|
|
|
+ fileList := strings.Split(configName, ".")
|
|
|
|
|
+ if len(fileList) < 2 {
|
|
|
|
|
+ return fmt.Errorf("%s", "文件格式不正确")
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ configName = fileList[0]
|
|
|
|
|
+ fileExt := fileList[1]
|
|
|
|
|
+ var err error
|
|
|
|
|
+ if fileExt == "json" {
|
|
|
|
|
+ err = LoadConfigFromJson(configName, configPath)
|
|
|
|
|
+ } else if fileExt == "yaml" || fileExt == "yml" {
|
|
|
|
|
+ err = LoadConfigFromYaml(configName, configPath)
|
|
|
|
|
+ } else {
|
|
|
|
|
+ err = fmt.Errorf("%s", "不支持的文件格式")
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // 出错直接返回
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ return nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+// LoadConfigFromYaml 装载yaml类型的配置文件
|
|
|
|
|
+func LoadConfigFromYaml(configName, configPath string) error {
|
|
|
|
|
+ v = viper.New()
|
|
|
|
|
+ v.SetConfigName(configName)
|
|
|
|
|
+ v.AddConfigPath(configPath)
|
|
|
|
|
+ //设置配置文件类型
|
|
|
|
|
+ v.SetConfigType("yaml")
|
|
|
|
|
+
|
|
|
|
|
+ if err := v.ReadInConfig(); err != nil {
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return parseConfig()
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+// LoadConfigFromJson 装载json类型的配置文件
|
|
|
|
|
+func LoadConfigFromJson(configName, configPath string) error {
|
|
|
|
|
+ v = viper.New()
|
|
|
|
|
+ v.SetConfigName(configName)
|
|
|
|
|
+ v.AddConfigPath(configPath)
|
|
|
|
|
+
|
|
|
|
|
+ //设置配置文件类型
|
|
|
|
|
+ v.SetConfigType("json")
|
|
|
|
|
+ if err := v.ReadInConfig(); err != nil {
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return parseConfig()
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func parseConfig() error {
|
|
|
|
|
+ Conf = &Configure{}
|
|
|
|
|
+ if err := v.Unmarshal(Conf); err != nil {
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+func commonVerify(image string) string {
|
|
|
|
|
+ config := map[string]interface{}{}
|
|
|
|
|
+ config["image"] = image
|
|
|
|
|
+ if Conf.VerifyType == ""{
|
|
|
|
|
+ config["type"] = "88888"
|
|
|
|
|
+ }else{
|
|
|
|
|
+ config["type"] = Conf.VerifyType
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ config["token"] = Conf.VerifyToken
|
|
|
|
|
+ //config["token"] = "2EEYFMjDVpMTky-Z_BCgt_I14g4qq_D63S3NsQesfMc"
|
|
|
|
|
+ configData, _ := json.Marshal(config)
|
|
|
|
|
+ body := bytes.NewBuffer([]byte(configData))
|
|
|
|
|
+ resp, err := http.Post("http://api.jfbym.com/api/YmServer/customApi", "application/json;charset=utf-8", body)
|
|
|
|
|
+ defer resp.Body.Close()
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return ""
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ data, _ := ioutil.ReadAll(resp.Body)
|
|
|
|
|
+ //fmt.Println(string(data), err)
|
|
|
|
|
+ return string(data)
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func codeGetNew(data []byte, account string) string {
|
|
|
|
|
+ name := account + ".png"
|
|
|
|
|
+ ioutil.WriteFile(name, data, 0666)
|
|
|
|
|
+ defer func() {
|
|
|
|
|
+ os.Remove(name)
|
|
|
|
|
+ }()
|
|
|
|
|
+
|
|
|
|
|
+ file, _ := os.Open(name)
|
|
|
|
|
+ defer file.Close()
|
|
|
|
|
+
|
|
|
|
|
+ fileInfo, _ := file.Stat()
|
|
|
|
|
+ imageSize := fileInfo.Size()
|
|
|
|
|
+ imageData := make([]byte, imageSize)
|
|
|
|
|
+ _, err := file.Read(imageData)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("保存文件失败")
|
|
|
|
|
+ return ""
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ base64Data := base64.StdEncoding.EncodeToString(imageData)
|
|
|
|
|
+ fmt.Println("开始识别验证码")
|
|
|
|
|
+ respData := commonVerify(base64Data)
|
|
|
|
|
+ //fmt.Println("验证码识别结果：", respData)
|
|
|
|
|
+ code := gjson.Get(respData, "code").Int()
|
|
|
|
|
+ if code != 10000 {
|
|
|
|
|
+ fmt.Println("验证码识别失败:", gjson.Get(respData, "msg").String())
|
|
|
|
|
+ return ""
|
|
|
|
|
+ }
|
|
|
|
|
+ return gjson.Get(respData, "data.data").String()
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+type CourseModule struct {
|
|
|
|
|
+ Name string
|
|
|
|
|
+ ModuleId string
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func getCourseModules(courceId string, cookie string, specialCourse bool) ([]CourseModule, error) {
|
|
|
|
|
+ url := fmt.Sprintf("https://lms.ouchn.cn/api/courses/%s/modules", courceId)
|
|
|
|
|
+ request, err := http.NewRequest("GET", url, nil)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return nil, err
|
|
|
|
|
+ }
|
|
|
|
|
+ //fmt.Println("URL :",url)
|
|
|
|
|
+ //fmt.Println("COOKIE :",cookie)
|
|
|
|
|
+
|
|
|
|
|
+ request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
|
|
|
+ request.Header.Set("Cookie", cookie)
|
|
|
|
|
+ request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
|
|
|
|
|
+ //request.Header.Set("Host", "chengdu.ouchn.cn")
|
|
|
|
|
+
|
|
|
|
|
+ client := http.Client{}
|
|
|
|
|
+ resp, err := client.Do(request)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return nil, err
|
|
|
|
|
+ }
|
|
|
|
|
+ defer resp.Body.Close()
|
|
|
|
|
+
|
|
|
|
|
+ respBytes, err := ioutil.ReadAll(resp.Body)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return nil, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ //fmt.Println("MODEL :",string(respBytes), err)
|
|
|
|
|
+
|
|
|
|
|
+ ret := []CourseModule{}
|
|
|
|
|
+ mos := gjson.GetBytes(respBytes, "modules").Array()
|
|
|
|
|
+ for _, mo := range mos {
|
|
|
|
|
+ /*if (len(mo.Get("syllabuses").Array())) == 0 && !specialCourse {
|
|
|
|
|
+ continue
|
|
|
|
|
+ }*/
|
|
|
|
|
+ item := CourseModule{}
|
|
|
|
|
+ item.ModuleId = fmt.Sprintf("%d", mo.Get("id").Int())
|
|
|
|
|
+ item.Name = mo.Get("name").String()
|
|
|
|
|
+ ret = append(ret, item)
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return ret, nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+type Course struct {
|
|
|
|
|
+ Name string
|
|
|
|
|
+ Url string
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func getCourceNew(ctx context.Context) []Course {
|
|
|
|
|
+ // https://lms.ouchn.cn/user/courses
|
|
|
|
|
+ courseList := []Course{}
|
|
|
|
|
+ err := chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Navigate("https://lms.ouchn.cn/user/courses"),
|
|
|
|
|
+ //chromedp.Sleep(2*time.Second),
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("进入学习网失败：",err)
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ cookie := getCookie(ctx)
|
|
|
|
|
+
|
|
|
|
|
+ i := int64(1)
|
|
|
|
|
+ totalPage, err := getCourceNewImpl(&courseList,cookie,i)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("获取课程失败;",err,"page=",i)
|
|
|
|
|
+ return courseList
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ //fmt.Println("totalPage:",totalPage)
|
|
|
|
|
+
|
|
|
|
|
+ if i < totalPage{
|
|
|
|
|
+ for i=2;i<=totalPage;i++{
|
|
|
|
|
+ _,err = getCourceNewImpl(&courseList,cookie,i )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("获取课程失败;",err,"page=",i)
|
|
|
|
|
+ continue
|
|
|
|
|
+ //return err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ return courseList
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func getCourceNewImpl(courses *[]Course,cookie string,page int64) (int64, error) {
|
|
|
|
|
+ url := fmt.Sprintf(`https://lms.ouchn.cn/api/my-courses?conditions={"status":["ongoing"],"keyword":""}&fields=id,name,course_code,department(id,name),grade(id,name),klass(id,name),course_type,cover,small_cover,start_date,end_date,is_started,is_closed,academic_year_id,semester_id,credit,compulsory,second_name,display_name,created_user(id,name),org(is_enterprise_or_organization),org_id,public_scope,course_attributes(teaching_class_name,copy_status,tip,data),audit_status,audit_remark,can_withdraw_course,imported_from,allow_clone,is_instructor,is_team_teaching,academic_year(id,name),semester(id,name),instructors(id,name,email,avatar_small_url),is_master,is_child,has_synchronized,master_course(name)&page=%d&page_size=10`, page)
|
|
|
|
|
+ //fmt.Println("获取课程 url:",url)
|
|
|
|
|
+ request, err := http.NewRequest("GET", url, nil)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return 0, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
|
|
|
+ request.Header.Set("Cookie", cookie)
|
|
|
|
|
+ request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
|
|
|
|
|
+ //request.Header.Set("Host", "chengdu.ouchn.cn")
|
|
|
|
|
+
|
|
|
|
|
+ client := http.Client{}
|
|
|
|
|
+ resp, err := client.Do(request)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return 0, err
|
|
|
|
|
+ }
|
|
|
|
|
+ if resp.StatusCode != http.StatusOK {
|
|
|
|
|
+ return 0, fmt.Errorf("wrong status code: %d", resp.StatusCode)
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ defer resp.Body.Close()
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ respBytes, err := ioutil.ReadAll(resp.Body)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return 0, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ //fmt.Println("RESP:",string(respBytes))
|
|
|
|
|
+ list := gjson.GetBytes(respBytes,"courses").Array()
|
|
|
|
|
+ for _,course := range list{
|
|
|
|
|
+ cou := Course{}
|
|
|
|
|
+ cou.Name = course.Get("display_name").String()
|
|
|
|
|
+ if cou.Name == ""{
|
|
|
|
|
+ cou.Name = course.Get("name").String()
|
|
|
|
|
+ }
|
|
|
|
|
+ //fmt.Println("DIS NAME :",course.Get("display_name").String(),"NAME:",course.Get("name").String())
|
|
|
|
|
+ //fmt.Println("course:",course.String())
|
|
|
|
|
+ cou.Url = course.Get("id").String()
|
|
|
|
|
+ if cou.Url != ""{
|
|
|
|
|
+ cou.Url = fmt.Sprintf("https://lms.ouchn.cn/course/%s/ng",cou.Url)
|
|
|
|
|
+ *courses = append(*courses,cou)
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ total_page := gjson.GetBytes(respBytes,"pages").Int()
|
|
|
|
|
+ return total_page, nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func getCourse(courses *[]Course,cookie string,page int64) (int64, error) {
|
|
|
|
|
+ url := fmt.Sprintf("https://menhu.pt.ouchn.cn/ouchnapp/wap/course/xskc-pc?page=%d", page)
|
|
|
|
|
+ fmt.Println("获取课程 :",url)
|
|
|
|
|
+ request, err := http.NewRequest("GET", url, nil)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return 0, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
|
|
|
+ request.Header.Set("Cookie", cookie)
|
|
|
|
|
+ request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36")
|
|
|
|
|
+ //request.Header.Set("Host", "chengdu.ouchn.cn")
|
|
|
|
|
+
|
|
|
|
|
+ client := http.Client{}
|
|
|
|
|
+ resp, err := client.Do(request)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return 0, err
|
|
|
|
|
+ }
|
|
|
|
|
+ defer resp.Body.Close()
|
|
|
|
|
+
|
|
|
|
|
+ respBytes, err := ioutil.ReadAll(resp.Body)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return 0, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ //fmt.Println("RESP :",string(respBytes))
|
|
|
|
|
+
|
|
|
|
|
+ code := gjson.GetBytes(respBytes, "e").Int()
|
|
|
|
|
+ if code == 0 {
|
|
|
|
|
+ data := gjson.GetBytes(respBytes, "d").String()
|
|
|
|
|
+ list := gjson.Get(data,"list").Array()
|
|
|
|
|
+ for _,course := range list{
|
|
|
|
|
+ cou := Course{}
|
|
|
|
|
+ cou.Name = course.Get("name").String()
|
|
|
|
|
+ cou.Url = course.Get("url").String()
|
|
|
|
|
+ *courses = append(*courses,cou)
|
|
|
|
|
+ }
|
|
|
|
|
+ total_page := gjson.Get(data,"total_page").Int()
|
|
|
|
|
+ return total_page, nil
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return 0, fmt.Errorf("code is not 0 ")
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func getCourseId(src string) string {
|
|
|
|
|
+ array := strings.Split(src, "/")
|
|
|
|
|
+ if len(array) < 5 {
|
|
|
|
|
+ return ""
|
|
|
|
|
+ }
|
|
|
|
|
+ return array[4]
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func getCookie(ctx context.Context)string {
|
|
|
|
|
+ // 执行登录操作
|
|
|
|
|
+ var cookies []*network.Cookie
|
|
|
|
|
+ err := chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.ActionFunc(func(ctx context.Context) error {
|
|
|
|
|
+ // 获取页面的 cookie
|
|
|
|
|
+ var err error
|
|
|
|
|
+ cookies, err = network.GetCookies().Do(ctx)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("获取cookie 失败：",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ return nil
|
|
|
|
|
+ }),
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return ""
|
|
|
|
|
+ }
|
|
|
|
|
+ ret := ""
|
|
|
|
|
+ for _, v := range cookies {
|
|
|
|
|
+ item := fmt.Sprintf("%s=%s", v.Name, v.Value)
|
|
|
|
|
+ if ret == "" {
|
|
|
|
|
+ ret = item
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ ret = ret + ";"
|
|
|
|
|
+ ret = ret + item
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ return ret
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+type ModuleSubInfo struct {
|
|
|
|
|
+ Name string
|
|
|
|
|
+ Id string
|
|
|
|
|
+ Stype string
|
|
|
|
|
+ SyllabusId int64
|
|
|
|
|
+ Type int // 1 page,2 vedio
|
|
|
|
|
+ PreUnCompelte bool
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+type ModuleInfo struct {
|
|
|
|
|
+ TxtInfos []ModuleSubInfo
|
|
|
|
|
+ ExamInfos []ModuleSubInfo
|
|
|
|
|
+ VedioInfos []ModuleSubInfo
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func GetModuleInfo(courseId string, moduleId string, cookie string) (*ModuleInfo, error) {
|
|
|
|
|
+ url := fmt.Sprintf("https://lms.ouchn.cn/api/course/%s"+
|
|
|
|
|
+ "/all-activities?module_ids=[%s]"+
|
|
|
|
|
+ "&activity_types=learning_activities,exams,classrooms,"+
|
|
|
|
|
+ "live_records,rollcalls&no-loading-animation=true", courseId, moduleId)
|
|
|
|
|
+ request, err := http.NewRequest("GET", url, nil)
|
|
|
|
|
+ //fmt.Println("url:",url)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return nil, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
|
|
|
+ request.Header.Set("Cookie", cookie)
|
|
|
|
|
+ request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
|
|
|
|
|
+ //request.Header.Set("Host", "chengdu.ouchn.cn")
|
|
|
|
|
+
|
|
|
|
|
+ client := http.Client{Timeout: 60 * time.Second}
|
|
|
|
|
+ resp, err := client.Do(request)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return nil, err
|
|
|
|
|
+ }
|
|
|
|
|
+ defer resp.Body.Close()
|
|
|
|
|
+
|
|
|
|
|
+ respBytes, err := ioutil.ReadAll(resp.Body)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return nil, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ //fmt.Println("resp:",string(respBytes))
|
|
|
|
|
+ ret := &ModuleInfo{}
|
|
|
|
|
+ // 试题
|
|
|
|
|
+ exams := []ModuleSubInfo{}
|
|
|
|
|
+ examsArray := gjson.GetBytes(respBytes, "exams").Array()
|
|
|
|
|
+ for _, v := range examsArray {
|
|
|
|
|
+ item := ModuleSubInfo{}
|
|
|
|
|
+ item.Name = v.Get("title").String()
|
|
|
|
|
+ unikey := v.Get("unique_key").String()
|
|
|
|
|
+ array := strings.Split(unikey, "-")
|
|
|
|
|
+ if len(array) < 2 {
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ item.Id = array[1]
|
|
|
|
|
+ exams = append(exams, item)
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // 文档和视频
|
|
|
|
|
+ texts := []ModuleSubInfo{}
|
|
|
|
|
+ vedios := []ModuleSubInfo{}
|
|
|
|
|
+ array := gjson.GetBytes(respBytes, "learning_activities").Array()
|
|
|
|
|
+ for _, v := range array {
|
|
|
|
|
+ stype := v.Get("type").String()
|
|
|
|
|
+ syllabusId := v.Get("syllabus_id").Int()
|
|
|
|
|
+ item := ModuleSubInfo{}
|
|
|
|
|
+
|
|
|
|
|
+ item.Name = v.Get("title").String()
|
|
|
|
|
+ item.SyllabusId = syllabusId
|
|
|
|
|
+ item.Stype = stype
|
|
|
|
|
+ unikey := v.Get("unique_key").String()
|
|
|
|
|
+ array := strings.Split(unikey, "-")
|
|
|
|
|
+ if len(array) < 2 {
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ prerequisitesArray := v.Get("prerequisites").Array()
|
|
|
|
|
+ preUnCompletion := false
|
|
|
|
|
+ if len(prerequisitesArray) == 0 || prerequisitesArray== nil{
|
|
|
|
|
+ preUnCompletion = true
|
|
|
|
|
+ }else{
|
|
|
|
|
+ for _,pre:= range prerequisitesArray{
|
|
|
|
|
+ isPreCompltetion := pre.Get("completion_criterion.completion_key").String()
|
|
|
|
|
+ if isPreCompltetion == "not_completed"{
|
|
|
|
|
+ preUnCompletion = true
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ item.PreUnCompelte = preUnCompletion
|
|
|
|
|
+ item.Id = array[1]
|
|
|
|
|
+ if stype == "page" {
|
|
|
|
|
+ item.Type = 1
|
|
|
|
|
+ texts = append(texts, item)
|
|
|
|
|
+ vedios = append(vedios, item)
|
|
|
|
|
+ } else if stype == "online_video" || (stype == "web_link" && strings.Contains(item.Name, "视频")) || stype == "material" {
|
|
|
|
|
+ index := -1
|
|
|
|
|
+ item.Type = 2
|
|
|
|
|
+ for i := len(vedios) - 1; i >= 0; i-- {
|
|
|
|
|
+ if item.SyllabusId == vedios[i].SyllabusId {
|
|
|
|
|
+ index = i
|
|
|
|
|
+ break
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ if index == -1 || index == len(vedios)-1 {
|
|
|
|
|
+ vedios = append(vedios, item)
|
|
|
|
|
+ } else {
|
|
|
|
|
+ tmp := []ModuleSubInfo{}
|
|
|
|
|
+ tmp = append(tmp, vedios[:index+1]...)
|
|
|
|
|
+ tmp = append(tmp, item)
|
|
|
|
|
+ tmp = append(tmp, vedios[index+1:]...)
|
|
|
|
|
+ vedios = tmp
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ ret.ExamInfos = exams
|
|
|
|
|
+ ret.TxtInfos = texts
|
|
|
|
|
+ ret.VedioInfos = vedios
|
|
|
|
|
+ return ret, nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func getUploadIds(moduleId string, cookie string) ([]int64, error) {
|
|
|
|
|
+ url := fmt.Sprintf("https://lms.ouchn.cn/api/activities/%s", moduleId)
|
|
|
|
|
+ request, err := http.NewRequest("GET", url, nil)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return nil, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
|
|
|
+ request.Header.Set("Cookie", cookie)
|
|
|
|
|
+ request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
|
|
|
|
|
+ //request.Header.Set("Host", "chengdu.ouchn.cn")
|
|
|
|
|
+
|
|
|
|
|
+ client := http.Client{}
|
|
|
|
|
+ resp, err := client.Do(request)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return nil, err
|
|
|
|
|
+ }
|
|
|
|
|
+ defer resp.Body.Close()
|
|
|
|
|
+
|
|
|
|
|
+ respBytes, err := ioutil.ReadAll(resp.Body)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return nil, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ ret := []int64{}
|
|
|
|
|
+ uploads := gjson.GetBytes(respBytes, "uploads").Array()
|
|
|
|
|
+ for _, upload := range uploads {
|
|
|
|
|
+ id := upload.Get("id").Int()
|
|
|
|
|
+ ret = append(ret, id)
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return ret, nil
|
|
|
|
|
+}
|
|
|
|
|
+func finishUploadId(uploadId int64, moduleId string, cookie string) error {
|
|
|
|
|
+ rBody := make(map[string]int64)
|
|
|
|
|
+ rBody["upload_id"] = uploadId
|
|
|
|
|
+ rByte, _ := json.Marshal(rBody)
|
|
|
|
|
+ url := fmt.Sprintf("https://lms.ouchn.cn/api/course/activities-read/%s", moduleId)
|
|
|
|
|
+ request, err := http.NewRequest("POST", url, bytes.NewBuffer(rByte))
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ request.Header.Set("Content-Type", "application/json")
|
|
|
|
|
+ request.Header.Set("Cookie", cookie)
|
|
|
|
|
+ request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
|
|
|
|
|
+ //request.Header.Set("Host", "chengdu.ouchn.cn")
|
|
|
|
|
+
|
|
|
|
|
+ client := http.Client{}
|
|
|
|
|
+ resp, err := client.Do(request)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ defer resp.Body.Close()
|
|
|
|
|
+
|
|
|
|
|
+ respBytes, err := ioutil.ReadAll(resp.Body)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ if true {
|
|
|
|
|
+ fmt.Println("do material resp:", string(respBytes))
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+func finishMaterial(uploadIds []int64, moduleId string, cookie string) {
|
|
|
|
|
+ for _, v := range uploadIds {
|
|
|
|
|
+ finishUploadId(v, moduleId, cookie)
|
|
|
|
|
+ time.Sleep(2 * time.Second)
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func doMaterial(moduleId string, cookie string) {
|
|
|
|
|
+ uploadIds, _ := getUploadIds(moduleId, cookie)
|
|
|
|
|
+ if len(uploadIds) > 0 {
|
|
|
|
|
+ finishMaterial(uploadIds, moduleId, cookie)
|
|
|
|
|
+ }
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func MakeTxtUrl(courseId string, moduleId string, id string) string {
|
|
|
|
|
+ return fmt.Sprintf("https://lms.ouchn.cn/course/%s/learning-activity/full-screen#/%s", courseId, id)
|
|
|
|
|
+}
|
|
|
|
|
+func MakeExamUrl(courseId string, moduleId string, id string) string {
|
|
|
|
|
+ return fmt.Sprintf("https://lms.ouchn.cn/course/%s/learning-activity/full-screen#/exam/%s", courseId, id)
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func VedioComplete(cookie string, id string, start, end int64) (int64, error) {
|
|
|
|
|
+ url := fmt.Sprintf("https://lms.ouchn.cn/api/course/activities-read/%s", id)
|
|
|
|
|
+ //fmt.Printf("get sections:%v,%v\n", str, cookie)
|
|
|
|
|
+ m := map[string]interface{}{
|
|
|
|
|
+ "start": start,
|
|
|
|
|
+ "end": end,
|
|
|
|
|
+ }
|
|
|
|
|
+ content, _ := json.Marshal(m)
|
|
|
|
|
+ reader := bytes.NewReader(content)
|
|
|
|
|
+ //fmt.Printf("post data:%s\n", content)
|
|
|
|
|
+ request, err := http.NewRequest("POST", url, reader)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return 0, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ request.Header.Set("Content-Type", "application/json")
|
|
|
|
|
+ request.Header.Set("Cookie", cookie)
|
|
|
|
|
+ request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
|
|
|
|
|
+
|
|
|
|
|
+ client := http.Client{}
|
|
|
|
|
+ resp, err := client.Do(request)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return 0, err
|
|
|
|
|
+ }
|
|
|
|
|
+ defer resp.Body.Close()
|
|
|
|
|
+
|
|
|
|
|
+ respBytes, err := ioutil.ReadAll(resp.Body)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return 0, err
|
|
|
|
|
+ }
|
|
|
|
|
+ fmt.Printf("get bytes:%s\n", respBytes)
|
|
|
|
|
+ if gjson.GetBytes(respBytes, "error_msg").String() != ""{
|
|
|
|
|
+ return 0,fmt.Errorf("视频处理失败(%s)",gjson.GetBytes(respBytes, "error_msg").String())
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if gjson.GetBytes(respBytes, "activity_id").Int() == 0 {
|
|
|
|
|
+ return 0, fmt.Errorf("视频处理失败")
|
|
|
|
|
+ }
|
|
|
|
|
+ str := gjson.GetBytes(respBytes, "data").Get("completeness").String()
|
|
|
|
|
+ if str == "full" {
|
|
|
|
|
+ return 100, nil
|
|
|
|
|
+ }
|
|
|
|
|
+ if str == "part" {
|
|
|
|
|
+ return 10, nil
|
|
|
|
|
+ }
|
|
|
|
|
+ return gjson.GetBytes(respBytes, "data").Get("completeness").Int(), nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+func VedioCompleteHandle(cookie string, id string) error {
|
|
|
|
|
+
|
|
|
|
|
+ count := 0
|
|
|
|
|
+ start, end := int64(0), int64(1000)
|
|
|
|
|
+ for count < 5 {
|
|
|
|
|
+ finish, err := VedioComplete(cookie, id, start, end)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ if finish < 100 {
|
|
|
|
|
+ start = end
|
|
|
|
|
+ end += 1000
|
|
|
|
|
+ count++
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ return nil
|
|
|
|
|
+ }
|
|
|
|
|
+ return fmt.Errorf("视频处理失败")
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func loadAnswerFromExcel() (map[string]AnswerInfo,error) {
|
|
|
|
|
+ answerMap := make(map[string]AnswerInfo)
|
|
|
|
|
+
|
|
|
|
|
+ entries, err := os.ReadDir("./")
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("Error reading directory:", err)
|
|
|
|
|
+ return answerMap, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ for _,v := range entries{
|
|
|
|
|
+ if !v.IsDir(){
|
|
|
|
|
+ if strings.HasPrefix(v.Name(),"account") || strings.HasPrefix(v.Name(),"~") || strings.HasPrefix(v.Name(),".") {
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ if strings.HasSuffix(v.Name(),".xlsx") || strings.HasSuffix(v.Name(),".xls") {
|
|
|
|
|
+ fmt.Println("读取文件:",v.Name())
|
|
|
|
|
+ file, err := excelize.OpenFile(v.Name())
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("open file err:", err)
|
|
|
|
|
+ return answerMap,err
|
|
|
|
|
+ }
|
|
|
|
|
+ for index, sheetName := range file.GetSheetList() {
|
|
|
|
|
+ if index > 0 {
|
|
|
|
|
+ break
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ rows, err := file.GetRows(sheetName)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return answerMap,err
|
|
|
|
|
+ }
|
|
|
|
|
+ for _, row := range rows {
|
|
|
|
|
+ rowLen := len(row)
|
|
|
|
|
+ if rowLen <= 2 {
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ for rowIndex,_:= range row{
|
|
|
|
|
+ row[rowIndex] = strings.TrimSpace(row[rowIndex])
|
|
|
|
|
+ if row[rowIndex] == ""{
|
|
|
|
|
+ fmt.Println("答案格式不正确，存在空列:",rowIndex,row)
|
|
|
|
|
+ return answerMap,fmt.Errorf("答案格式不正确，存在空列")
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ tmpAnswer := AnswerInfo{}
|
|
|
|
|
+ tmpAnswer.Question = row[1]
|
|
|
|
|
+ tmpAnswer.Question = ParseHan(tmpAnswer.Question)
|
|
|
|
|
+ //tmpAnswer.InfoName = row[1]
|
|
|
|
|
+ tmpAnswer.Type = row[0]
|
|
|
|
|
+ tmpAnswer.Answer = row[2:]
|
|
|
|
|
+ key := tmpAnswer.Type+":"+tmpAnswer.Question
|
|
|
|
|
+ key = strings.ReplaceAll(key, " ", "")
|
|
|
|
|
+ answerMap[key] = tmpAnswer
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ //fmt.Println(v.Name(),v.IsDir())
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ if len(answerMap) == 0 {
|
|
|
|
|
+ return answerMap,fmt.Errorf("答案为空")
|
|
|
|
|
+ }
|
|
|
|
|
+ return answerMap,nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func handleVideo(ctx context.Context,moduleInfo *ModuleInfo, module CourseModule,courseId string,cookie string) error {
|
|
|
|
|
+ vedioLen := len(moduleInfo.VedioInfos)
|
|
|
|
|
+
|
|
|
|
|
+ for index, info := range moduleInfo.VedioInfos {
|
|
|
|
|
+ if Conf.SkipProcessed{
|
|
|
|
|
+ nextIndex := index+1
|
|
|
|
|
+ if index+1 < vedioLen{
|
|
|
|
|
+ if !moduleInfo.VedioInfos[nextIndex].PreUnCompelte{
|
|
|
|
|
+ fmt.Println("已处理("+info.Name+")跳过")
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if info.Type == 1{
|
|
|
|
|
+ if Conf.HandlePage{
|
|
|
|
|
+ fmt.Printf("------------开始处理网页数据：%s------------\n", info.Name)
|
|
|
|
|
+ url := MakeTxtUrl(courseId, module.ModuleId, info.Id)
|
|
|
|
|
+ err := chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Navigate(url),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("跳转网页:",url,"失败:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ time.Sleep(time.Duration(Conf.HandleInterval) * time.Second)
|
|
|
|
|
+ }
|
|
|
|
|
+ }else if info.Type == 2 {
|
|
|
|
|
+ if info.Stype == "material" {
|
|
|
|
|
+ if Conf.HandleMaterial{
|
|
|
|
|
+ fmt.Printf("------------开始处理材料数据：%s------------\n", info.Name)
|
|
|
|
|
+ url := MakeTxtUrl(courseId, module.ModuleId, info.Id)
|
|
|
|
|
+ err := chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Navigate(url),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("跳转网页:",url,"失败:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ doMaterial(info.Id, cookie)
|
|
|
|
|
+ time.Sleep(time.Duration(Conf.HandleInterval) * time.Second)
|
|
|
|
|
+ }
|
|
|
|
|
+ }else{
|
|
|
|
|
+ if Conf.HandleVideo{
|
|
|
|
|
+ fmt.Printf("------------开始处理视频数据：%s------------\n", info.Name)
|
|
|
|
|
+ url := MakeTxtUrl(courseId, module.ModuleId, info.Id)
|
|
|
|
|
+ err := chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Navigate(url),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("跳转网页:",url,"失败:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ err = VedioCompleteHandle(cookie, info.Id)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Printf("视频%s处理失败（%s）\n", info.Name, err.Error())
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ time.Sleep(time.Duration(Conf.HandleInterval) * time.Second)
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ }
|
|
|
|
|
+ return nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func handleCrawlerChooseImpl(ctx context.Context,node *cdp.Node,answerType string )(AnswerInfo,error){
|
|
|
|
|
+ answerInfo := AnswerInfo{}
|
|
|
|
|
+ answerInfo.Type = answerType
|
|
|
|
|
+ var childNodes []*cdp.Node
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/div/span/p", &childNodes)); err != nil {
|
|
|
|
|
+ fmt.Println("获取NODE失败：",err)
|
|
|
|
|
+ return answerInfo,err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ var question string
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Text(childNodes[0].FullXPath(), &question)); err != nil {
|
|
|
|
|
+ fmt.Println("获取问题失败:", err)
|
|
|
|
|
+ return answerInfo,err
|
|
|
|
|
+ }
|
|
|
|
|
+ question = strings.TrimSpace(question)
|
|
|
|
|
+ fmt.Println("问题:", question)
|
|
|
|
|
+ answerInfo.Question = question
|
|
|
|
|
+
|
|
|
|
|
+ var childNodes1 []*cdp.Node
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/ol/li/label/div/span", &childNodes1)); err != nil {
|
|
|
|
|
+ fmt.Println("获取NODE失败：",err)
|
|
|
|
|
+ return answerInfo,err
|
|
|
|
|
+ }
|
|
|
|
|
+ var tmpContent string
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Text(childNodes1[0].FullXPath(), &tmpContent)); err != nil {
|
|
|
|
|
+ fmt.Println("获取答案失败:", err)
|
|
|
|
|
+ return answerInfo,err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if tmpContent == ""{
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/ol/li/label/div/span/p", &childNodes1)); err != nil {
|
|
|
|
|
+ fmt.Println("获取NODE失败：",err)
|
|
|
|
|
+ return answerInfo,err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ for _,v := range childNodes1{
|
|
|
|
|
+ pContent := ""
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Text(v.FullXPath(), &pContent)); err != nil {
|
|
|
|
|
+ fmt.Println("获取答案失败:", err)
|
|
|
|
|
+ return answerInfo,err
|
|
|
|
|
+ }
|
|
|
|
|
+ pContent = strings.TrimSpace(pContent)
|
|
|
|
|
+ answerInfo.Answer = append(answerInfo.Answer,pContent)
|
|
|
|
|
+ }
|
|
|
|
|
+ return answerInfo,nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func handleCrawlerAnswerImpl(ctx context.Context,node *cdp.Node)(AnswerInfo,error){
|
|
|
|
|
+ answerInfo := AnswerInfo{}
|
|
|
|
|
+ answerInfo.Type = "问答题"
|
|
|
|
|
+ var childNodes []*cdp.Node
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div[1]/div[1]/span/p[1]", &childNodes)); err != nil {
|
|
|
|
|
+ fmt.Println("获取NODE失败：",err)
|
|
|
|
|
+ return answerInfo ,err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ var question string
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Text(childNodes[0].FullXPath(), &question)); err != nil {
|
|
|
|
|
+ fmt.Println("获取问题失败:", err)
|
|
|
|
|
+ return answerInfo ,err
|
|
|
|
|
+ }
|
|
|
|
|
+ question = strings.TrimSpace(question)
|
|
|
|
|
+ fmt.Println("问题:", question)
|
|
|
|
|
+ //fmt.Println(answerMap)
|
|
|
|
|
+ answerInfo.Question = question
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ return answerInfo, nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func handleAnswerImpl(ctx context.Context,node *cdp.Node,answerMap map[string]AnswerInfo,infoName string )(int,error){
|
|
|
|
|
+ isChoose := 1
|
|
|
|
|
+ var childNodes []*cdp.Node
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div[1]/div[1]/span/p[1]", &childNodes)); err != nil {
|
|
|
|
|
+ fmt.Println("获取NODE失败：",err)
|
|
|
|
|
+ return isChoose ,err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ var question string
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Text(childNodes[0].FullXPath(), &question)); err != nil {
|
|
|
|
|
+ fmt.Println("获取问题失败:", err)
|
|
|
|
|
+ return isChoose ,err
|
|
|
|
|
+ }
|
|
|
|
|
+ question = strings.TrimSpace(question)
|
|
|
|
|
+ question = ParseHan(question)
|
|
|
|
|
+ fmt.Println("问题:", question)
|
|
|
|
|
+ infoName = strings.TrimSpace(infoName)
|
|
|
|
|
+ key := "问答题"+":"+question
|
|
|
|
|
+ key = strings.ReplaceAll(key, " ", "")
|
|
|
|
|
+
|
|
|
|
|
+ tmpAnswer,ok := answerMap[key]
|
|
|
|
|
+ if !ok{
|
|
|
|
|
+ fmt.Println("问题:",question," 未提供答案")
|
|
|
|
|
+ return isChoose,fmt.Errorf("无答案")
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if len(tmpAnswer.Answer) == 0 {
|
|
|
|
|
+ fmt.Println("问题:",question," 未提供答案")
|
|
|
|
|
+ return isChoose,fmt.Errorf("无答案")
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ escapedAnswer := strconv.Quote(tmpAnswer.Answer[0])
|
|
|
|
|
+ if len(tmpAnswer.Answer) > 1 {
|
|
|
|
|
+ escapedAnswer = strconv.Quote(tmpAnswer.Answer[random(len(tmpAnswer.Answer))])
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if len(escapedAnswer) == 0 {
|
|
|
|
|
+ fmt.Println("问题:",question," 无答案")
|
|
|
|
|
+ return isChoose,fmt.Errorf("无答案")
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ jsCode := fmt.Sprintf(`
|
|
|
|
|
+ var element = document.evaluate("%s", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
|
|
|
|
|
+ if (element) {
|
|
|
|
|
+ element.textContent = %s;
|
|
|
|
|
+ }
|
|
|
|
|
+ `,node.FullXPath()+"/div/div/div/div/div/div/div/p",escapedAnswer)
|
|
|
|
|
+ //fmt.Println("JS CODE:",jsCode)
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Evaluate(jsCode, nil)); err != nil {
|
|
|
|
|
+ fmt.Println("设置 <p> 元素文本内容失败：", err)
|
|
|
|
|
+ return isChoose, err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return 0, nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func random(count int) int {
|
|
|
|
|
+ if count == 0 {
|
|
|
|
|
+ return 0
|
|
|
|
|
+ }
|
|
|
|
|
+ //rand.Seed(time.Now().UnixNano())
|
|
|
|
|
+ return rand.Intn(count)
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func handleChooseImpl(ctx context.Context,node *cdp.Node,answerMap map[string]AnswerInfo,isMulti bool,typeStr string,randomWrong bool ,infoName string)(int,error){
|
|
|
|
|
+ miss := 1
|
|
|
|
|
+ var childNodes []*cdp.Node
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/div/span/p", &childNodes)); err != nil {
|
|
|
|
|
+ fmt.Println("获取NODE失败：",err)
|
|
|
|
|
+ return miss ,err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ var question string
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Text(childNodes[0].FullXPath(), &question)); err != nil {
|
|
|
|
|
+ fmt.Println("获取问题失败:", err)
|
|
|
|
|
+ return miss ,err
|
|
|
|
|
+ }
|
|
|
|
|
+ question = strings.TrimSpace(question)
|
|
|
|
|
+ question = ParseHan(question)
|
|
|
|
|
+ fmt.Println("问题:", question,"是否随机出错：",randomWrong)
|
|
|
|
|
+ infoName = strings.TrimSpace(infoName)
|
|
|
|
|
+ key := typeStr+":"+question
|
|
|
|
|
+ key = strings.ReplaceAll(key, " ", "")
|
|
|
|
|
+ fmt.Println("KEY :",key)
|
|
|
|
|
+ tmpAnswer,ok := answerMap[key]
|
|
|
|
|
+ if !ok{
|
|
|
|
|
+ fmt.Println("问题:",question," 未提供答案")
|
|
|
|
|
+ //return isChoose,nil
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ time.Sleep(1*time.Second)
|
|
|
|
|
+
|
|
|
|
|
+ if isMulti{
|
|
|
|
|
+ var nodes []*cdp.Node
|
|
|
|
|
+ err := chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Nodes(node.FullXPath()+"/div/div[2]/ol/li/label/span/input", &nodes, chromedp.BySearch),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("勾选组件不存在:",err)
|
|
|
|
|
+ return miss,err
|
|
|
|
|
+ }
|
|
|
|
|
+ for index,v := range nodes{
|
|
|
|
|
+ attr := v.AttributeValue("class")
|
|
|
|
|
+ //fmt.Println("第",index+1," class:",attr)
|
|
|
|
|
+ if strings.Contains(attr, "ng-not-empty") {
|
|
|
|
|
+ fmt.Println("第",index+1,"项非空取消选择")
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Click(v.FullXPath())); err != nil {
|
|
|
|
|
+ fmt.Println("点击取消失败：",err)
|
|
|
|
|
+ return miss,err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ var childNodes1 []*cdp.Node
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/ol/li/label/div/span", &childNodes1)); err != nil {
|
|
|
|
|
+ fmt.Println("获取NODE失败：",err)
|
|
|
|
|
+ return miss ,err
|
|
|
|
|
+ }
|
|
|
|
|
+ var tmpContent string
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Text(childNodes1[0].FullXPath(), &tmpContent)); err != nil {
|
|
|
|
|
+ fmt.Println("获取答案失败:", err)
|
|
|
|
|
+ return miss ,err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if tmpContent == ""{
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Nodes(node.FullXPath()+"/div/div/ol/li/label/div/span/p", &childNodes1)); err != nil {
|
|
|
|
|
+ fmt.Println("获取NODE失败：",err)
|
|
|
|
|
+ return miss ,err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ ///html/body/div[2]/div[4]/div[3]/div[8]/div/div/div[5]/div/div[2]/div/ol/li[18]/div/div[2]/ol/li[1]/label/div/span/p
|
|
|
|
|
+
|
|
|
|
|
+ //fmt.Println("node:",childNodes1)
|
|
|
|
|
+ rightIndex := -1
|
|
|
|
|
+ for nodeIndex,v := range childNodes1{
|
|
|
|
|
+ pContent := ""
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Text(v.FullXPath(), &pContent)); err != nil {
|
|
|
|
|
+ fmt.Println("获取答案失败:", err)
|
|
|
|
|
+ return miss ,err
|
|
|
|
|
+ }
|
|
|
|
|
+ pContent = strings.TrimSpace(pContent)
|
|
|
|
|
+ for _,ans := range tmpAnswer.Answer{
|
|
|
|
|
+ if ans == pContent{
|
|
|
|
|
+ miss = 0
|
|
|
|
|
+ fmt.Println("选择答案:", pContent)
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Click(v.FullXPath())); err != nil {
|
|
|
|
|
+ fmt.Println("点击失败：",err)
|
|
|
|
|
+ return miss, err
|
|
|
|
|
+ }
|
|
|
|
|
+ rightIndex = nodeIndex
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if miss == 1 {
|
|
|
|
|
+ index := random(len(childNodes1))
|
|
|
|
|
+ fmt.Println("未找到答案，随机选择第",index+1,"项")
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Click(childNodes1[index].FullXPath())); err != nil {
|
|
|
|
|
+ fmt.Println("点击失败：",err)
|
|
|
|
|
+ return miss,err
|
|
|
|
|
+ }
|
|
|
|
|
+ }else{
|
|
|
|
|
+ if randomWrong{
|
|
|
|
|
+ index := 0
|
|
|
|
|
+ for i:=0;i<10;i++ {
|
|
|
|
|
+ index = random(len(childNodes1))
|
|
|
|
|
+ if rightIndex == index{
|
|
|
|
|
+ continue
|
|
|
|
|
+ }else{
|
|
|
|
|
+ break
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Println("随机出错，随机选择第",index+1,"项")
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Click(childNodes1[index].FullXPath())); err != nil {
|
|
|
|
|
+ fmt.Println("点击失败：",err)
|
|
|
|
|
+ return miss,err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return miss, nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+type AnswerInfo struct{
|
|
|
|
|
+ Question string
|
|
|
|
|
+ Type string
|
|
|
|
|
+ InfoName string
|
|
|
|
|
+ Answer []string
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func checkExamFinish(id string, cookie string) (finish bool, submited bool, noScore bool) {
|
|
|
|
|
+ url := fmt.Sprintf("https://lms.ouchn.cn/api/exams/%s", id)
|
|
|
|
|
+ request, err := http.NewRequest("GET", url, nil)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return false, false, false
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
|
|
|
+ request.Header.Set("Cookie", cookie)
|
|
|
|
|
+ request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
|
|
|
|
|
+ //request.Header.Set("Host", "chengdu.ouchn.cn")
|
|
|
|
|
+
|
|
|
|
|
+ client := http.Client{}
|
|
|
|
|
+ resp, err := client.Do(request)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return false, false, false
|
|
|
|
|
+ }
|
|
|
|
|
+ defer resp.Body.Close()
|
|
|
|
|
+
|
|
|
|
|
+ respBytes, err := ioutil.ReadAll(resp.Body)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return false, false, false
|
|
|
|
|
+ }
|
|
|
|
|
+ scorePercentage := gjson.GetBytes(respBytes, "score_percentage").String()
|
|
|
|
|
+ if scorePercentage == "0" || scorePercentage == "0.0" || scorePercentage == "0.00" {
|
|
|
|
|
+ return true, false, true
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ submitTimes := gjson.GetBytes(respBytes, "submit_times").Int()
|
|
|
|
|
+ submittedTimes := gjson.GetBytes(respBytes, "submitted_times").Int()
|
|
|
|
|
+ if submittedTimes > 0 {
|
|
|
|
|
+ submited = true
|
|
|
|
|
+ }
|
|
|
|
|
+ if submitTimes == submittedTimes && submitTimes > 0 {
|
|
|
|
|
+ return true, submited, false
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return false, submited, false
|
|
|
|
|
+
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func checkExamScore(id string, cookie string) (finish bool) {
|
|
|
|
|
+ url := fmt.Sprintf("https://lms.ouchn.cn/api/exams/%s/submissions", id)
|
|
|
|
|
+ request, err := http.NewRequest("GET", url, nil)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return false
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
|
|
|
+ request.Header.Set("Cookie", cookie)
|
|
|
|
|
+ request.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.81 Safari/537.36")
|
|
|
|
|
+ //request.Header.Set("Host", "chengdu.ouchn.cn")
|
|
|
|
|
+
|
|
|
|
|
+ client := http.Client{}
|
|
|
|
|
+ resp, err := client.Do(request)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return false
|
|
|
|
|
+ }
|
|
|
|
|
+ defer resp.Body.Close()
|
|
|
|
|
+
|
|
|
|
|
+ respBytes, err := ioutil.ReadAll(resp.Body)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return false
|
|
|
|
|
+ }
|
|
|
|
|
+ examScore := gjson.GetBytes(respBytes, "exam_score").Int()
|
|
|
|
|
+ fmt.Println("分数：", examScore, " 最低要求分数：", Conf.MinScore)
|
|
|
|
|
+ if int(examScore) >= Conf.MinScore {
|
|
|
|
|
+ return true
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return false
|
|
|
|
|
+
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func writeExameToExcel(infos []AnswerInfo,writer *excelize.StreamWriter,lastLine int,infoName string)(int,error){
|
|
|
|
|
+ for _,v := range infos{
|
|
|
|
|
+ lastLine++
|
|
|
|
|
+ cell, err := excelize.CoordinatesToCellName(1,lastLine)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+ rowList := []interface{}{}
|
|
|
|
|
+ rowList = append(rowList,v.Type)
|
|
|
|
|
+ rowList = append(rowList,v.Question)
|
|
|
|
|
+ //rowList = append(rowList,infoName)
|
|
|
|
|
+ for _,ans := range v.Answer{
|
|
|
|
|
+ rowList = append(rowList,ans)
|
|
|
|
|
+ }
|
|
|
|
|
+ err = writer.SetRow(cell, rowList)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ return lastLine,nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+func createFile(fileName string) (*excelize.File, *excelize.StreamWriter) {
|
|
|
|
|
+ file := excelize.NewFile()
|
|
|
|
|
+ //设置表名
|
|
|
|
|
+ file.SetSheetName("Sheet1", "表1")
|
|
|
|
|
+ //创建流式写入
|
|
|
|
|
+ writer, err := file.NewStreamWriter("表1")
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return nil, nil
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ file.SaveAs(fileName)
|
|
|
|
|
+
|
|
|
|
|
+ return file, writer
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func pullBottom(ctx context.Context,needPull bool ) error {
|
|
|
|
|
+ if needPull{
|
|
|
|
|
+ err := waitVisibleTimeout(ctx,"/html/body/div/div/div/div/div[2]/div[2]/div[1]")
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("获取div失败:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ if err := chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Evaluate(`document.evaluate("/html/body/div/div/div/div/div[2]/div[2]/div[1]", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.scrollTop = document.evaluate("/html/body/div/div/div/div/div[2]/div[2]/div[1]", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.scrollHeight`, nil),
|
|
|
|
|
+ ); err != nil {
|
|
|
|
|
+ fmt.Println("拉动窗口失败：",err )
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ err := waitVisibleAndClickTimeout(ctx,"//a[@class='button button-green take-exam ng-scope']")
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("开始答题失败，找不到开始按钮:",err)
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ err = waitVisibleAndClickTimeout(ctx,"//input[@type='checkbox' and @name='confirm']")
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("开始答题失败，找不到确认按钮:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+// 爬取试题
|
|
|
|
|
+func crawlerExame(ctx context.Context,moduleInfo *ModuleInfo, module CourseModule,courseId string,writer *excelize.StreamWriter,lastLine int) (int,error) {
|
|
|
|
|
+ for _, info := range moduleInfo.ExamInfos {
|
|
|
|
|
+ info.Name = strings.TrimSpace(info.Name)
|
|
|
|
|
+ url := MakeExamUrl(courseId, module.ModuleId, info.Id)
|
|
|
|
|
+ err := chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Navigate(url),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Println("INFO NAME:",info.Name )
|
|
|
|
|
+ needPull := false
|
|
|
|
|
+ for i:=0 ;i<=3;i++{
|
|
|
|
|
+ err = pullBottom(ctx,needPull)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ needPull = true
|
|
|
|
|
+ time.Sleep(3*time.Second)
|
|
|
|
|
+ fmt.Println("第",i+1,"次失败：",err)
|
|
|
|
|
+ continue
|
|
|
|
|
+ }else{
|
|
|
|
|
+ break
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("开始答题失败:",err)
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ time.Sleep(1*time.Second)
|
|
|
|
|
+
|
|
|
|
|
+ err = waitVisibleAndClickTimeout(ctx,"//button[text()='开始答题' or text()='继续答题']")
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("开始答题失败:",err)
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ err = waitVisibleTimeout(ctx,"//li[contains(@class, 'subject ng-scope ')]")
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("等待题目出错:",err)
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ var questionClasses []*cdp.Node
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Nodes("//li[contains(@class, 'subject ng-scope ')]", &questionClasses)); err != nil {
|
|
|
|
|
+ fmt.Println("获取题目出错：",err)
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ answerInfoList := []AnswerInfo{}
|
|
|
|
|
+ fmt.Println(questionClasses,"------------------题目长度（包含题目类型）：",len(questionClasses))
|
|
|
|
|
+ for _, v := range questionClasses {
|
|
|
|
|
+ attr := v.AttributeValue("class")
|
|
|
|
|
+ if strings.Contains(attr, "true_or_false") {
|
|
|
|
|
+ fmt.Println("判断题")
|
|
|
|
|
+ answerInfo ,err := handleCrawlerChooseImpl(ctx,v,"判断题")
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+ answerInfoList = append(answerInfoList,answerInfo)
|
|
|
|
|
+ }else if strings.Contains(attr, "single_selection") {
|
|
|
|
|
+ fmt.Println("单选题")
|
|
|
|
|
+ answerInfo ,err := handleCrawlerChooseImpl(ctx,v,"单选题")
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+ answerInfoList = append(answerInfoList,answerInfo)
|
|
|
|
|
+ }else if strings.Contains(attr, "multiple_selection") {
|
|
|
|
|
+ fmt.Println("多选题")
|
|
|
|
|
+ answerInfo ,err := handleCrawlerChooseImpl(ctx,v,"多选题")
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+ answerInfoList = append(answerInfoList,answerInfo)
|
|
|
|
|
+ } else if strings.Contains(attr, "short_answer") {
|
|
|
|
|
+ fmt.Println("问答题")
|
|
|
|
|
+ answerInfo ,err := handleCrawlerAnswerImpl(ctx,v)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+ answerInfoList = append(answerInfoList,answerInfo)
|
|
|
|
|
+ }else{
|
|
|
|
|
+ fmt.Println("非题目")
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ lastLine, err = writeExameToExcel(answerInfoList,writer,lastLine,info.Name)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return lastLine,err
|
|
|
|
|
+ }
|
|
|
|
|
+ time.Sleep(time.Duration(Conf.HandleInterval)*time.Second)
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return lastLine,nil
|
|
|
|
|
+}
|
|
|
|
|
+func ParseHan(str string) string {
|
|
|
|
|
+ ret := ""
|
|
|
|
|
+ for _, r := range str {
|
|
|
|
|
+ if unicode.Is(unicode.Han, r) {
|
|
|
|
|
+ ret = ret + string(r)
|
|
|
|
|
+ } else if (r >= 48 && r <= 57) || (r >= 65 && r <= 90) || (r >= 97 && r <= 112) {
|
|
|
|
|
+ ret = ret + string(r)
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ return ret
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func handleExame(ctx context.Context,moduleInfo *ModuleInfo, module CourseModule,courseId string,cookie string,answerMap map[string]AnswerInfo) error {
|
|
|
|
|
+ for _, info := range moduleInfo.ExamInfos {
|
|
|
|
|
+ info.Name = strings.TrimSpace(info.Name)
|
|
|
|
|
+ fmt.Println("------------开始处理试题：",info.Name,"------------")
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ finsh, submited, noScore := checkExamFinish(info.Id, cookie)
|
|
|
|
|
+ if !Conf.HandelNoPoint{
|
|
|
|
|
+ if noScore {
|
|
|
|
|
+ fmt.Printf("不计分，该试题不做\n")
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if finsh {
|
|
|
|
|
+ fmt.Printf("试题达到最大提交次数\n")
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if submited {
|
|
|
|
|
+ isFinsh := checkExamScore(info.Id, cookie)
|
|
|
|
|
+ if isFinsh {
|
|
|
|
|
+ fmt.Printf("已完成该试题,并满足分数\n")
|
|
|
|
|
+ continue
|
|
|
|
|
+ } else {
|
|
|
|
|
+ fmt.Printf("已完成该试题,不满足分数,继续答题\n")
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ url := MakeExamUrl(courseId, module.ModuleId, info.Id)
|
|
|
|
|
+ err := chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Navigate(url),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ needPull := false
|
|
|
|
|
+ for i:=0 ;i<=3;i++{
|
|
|
|
|
+ err = pullBottom(ctx,needPull)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ needPull = true
|
|
|
|
|
+ time.Sleep(3*time.Second)
|
|
|
|
|
+ fmt.Println("第",i+1,"次失败：",err)
|
|
|
|
|
+ continue
|
|
|
|
|
+ }else{
|
|
|
|
|
+ break
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("开始答题失败:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ time.Sleep(1*time.Second)
|
|
|
|
|
+
|
|
|
|
|
+ err = waitVisibleAndClickTimeout(ctx,"//button[text()='开始答题' or text()='继续答题']")
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("开始答题失败:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ err = waitVisibleTimeout(ctx,"//li[contains(@class, 'subject ng-scope ')]")
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("等待题目出错:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ time.Sleep(3*time.Second)
|
|
|
|
|
+
|
|
|
|
|
+ //time.Sleep(10*time.Second)
|
|
|
|
|
+ var questionClasses []*cdp.Node
|
|
|
|
|
+ if err := chromedp.Run(ctx, chromedp.Nodes("//li[contains(@class, 'subject ng-scope ')]", &questionClasses)); err != nil {
|
|
|
|
|
+ fmt.Println("获取题目出错：",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ total := 0
|
|
|
|
|
+ miss := 0
|
|
|
|
|
+
|
|
|
|
|
+ randomCount := random(Conf.MaxRandomQuestion)
|
|
|
|
|
+ randomHandelCount := 0
|
|
|
|
|
+ fmt.Println("可出错个数：",randomCount,Conf.MaxRandomQuestion)
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Println("------------------题目长度（包含题目类型）：",len(questionClasses))
|
|
|
|
|
+ for _, v := range questionClasses {
|
|
|
|
|
+ //fmt.Println("题目")
|
|
|
|
|
+ randWrong := false
|
|
|
|
|
+ if randomCount > randomHandelCount{
|
|
|
|
|
+ isWrong := random(2)
|
|
|
|
|
+ //fmt.Println("isWrong:",isWrong)
|
|
|
|
|
+ if isWrong == 1{
|
|
|
|
|
+ randomHandelCount++
|
|
|
|
|
+ fmt.Println("已随机个数：",randomHandelCount)
|
|
|
|
|
+ randWrong = true
|
|
|
|
|
+ //fmt.Println("randWrong111111111111111",randWrong)
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ //fmt.Println("randWrong",randWrong)
|
|
|
|
|
+ attr := v.AttributeValue("class")
|
|
|
|
|
+ if strings.Contains(attr, "true_or_false") {
|
|
|
|
|
+ //fmt.Println("判断题")
|
|
|
|
|
+ isChoose ,err := handleChooseImpl(ctx,v,answerMap,false,"判断题",randWrong,info.Name)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ miss = miss + isChoose
|
|
|
|
|
+ }else if strings.Contains(attr, "single_selection") {
|
|
|
|
|
+ //fmt.Println("单选题")
|
|
|
|
|
+ isChoose ,err := handleChooseImpl(ctx,v,answerMap,false,"单选题",randWrong,info.Name)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ miss = miss + isChoose
|
|
|
|
|
+ }else if strings.Contains(attr, "multiple_selection") {
|
|
|
|
|
+ //fmt.Println("多选题")
|
|
|
|
|
+ isChoose ,err := handleChooseImpl(ctx,v,answerMap,true,"多选题",randWrong,info.Name)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ miss = miss + isChoose
|
|
|
|
|
+ } else if strings.Contains(attr, "short_answer") {
|
|
|
|
|
+ //fmt.Println("问答题")
|
|
|
|
|
+ isChoose ,err := handleAnswerImpl(ctx,v,answerMap,info.Name)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ miss = miss + isChoose
|
|
|
|
|
+ }else{
|
|
|
|
|
+ if randWrong {
|
|
|
|
|
+ randomHandelCount--
|
|
|
|
|
+ }
|
|
|
|
|
+ //fmt.Println("非题目")
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ total++
|
|
|
|
|
+
|
|
|
|
|
+ if miss > Conf.QuestionCount{
|
|
|
|
|
+ fmt.Println("无答案数(",miss,")超过设定值：",Conf.QuestionCount)
|
|
|
|
|
+ return fmt.Errorf("无答案数过多超过设定值")
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if miss > Conf.QuestionCount{
|
|
|
|
|
+ fmt.Println("无答案数(",miss,")超过设定值：",Conf.QuestionCount)
|
|
|
|
|
+ return fmt.Errorf("无答案数过多超过设定值")
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Println("等待：",Conf.CommitExamTime,"秒提交")
|
|
|
|
|
+ time.Sleep(time.Duration(Conf.CommitExamTime)*time.Second)
|
|
|
|
|
+
|
|
|
|
|
+ err = waitVisibleAndClickTimeout(ctx,"//a[@class='button button-green ng-scope' and text()='交卷']")
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("未找到交卷按钮:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ time.Sleep(2*time.Second)
|
|
|
|
|
+
|
|
|
|
|
+ nodes ,err := getNodeTimeout(ctx,"#submit-exam-confirmation-popup > div > div:nth-child(3) > div > button:nth-child(1)")
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("获取确认节点失败")
|
|
|
|
|
+ nodes ,err = getNodeTimeout(ctx,`*[@id="submit-exam-confirmation-popup"]/div/div[3]/div/button[1]`)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("获取确认节点失败")
|
|
|
|
|
+ nodes ,err = getNodeTimeout(ctx,`/html[1]/body[1]/div[10]/div[1]/div[3]/div[1]/button[1]`)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("获取确认节点失败")
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ for _, node := range nodes {
|
|
|
|
|
+ if node.FullXPath() == `/html[1]/body[1]/div[10]/div[1]/div[3]/div[1]/button[1]`{
|
|
|
|
|
+ fmt.Println("确定提交")
|
|
|
|
|
+ //time.Sleep(100*time.Hour)
|
|
|
|
|
+ err = chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Click(node.FullXPath(), chromedp.BySearch),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("提交答案失败：",err)
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Println("等待时间间隔（handleInterval）：",Conf.HandleInterval)
|
|
|
|
|
+ time.Sleep(time.Duration(Conf.HandleInterval)*time.Second)
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func waitVisibleTimeout(ctx context.Context,sel interface{}) error {
|
|
|
|
|
+ ctxTmp, cancleTmp := context.WithTimeout(ctx, time.Duration(Conf.Timeout)*time.Second)
|
|
|
|
|
+ defer cancleTmp()
|
|
|
|
|
+ done := make(chan struct{})
|
|
|
|
|
+ go func() {
|
|
|
|
|
+ defer close(done)
|
|
|
|
|
+ err := chromedp.Run(ctxTmp,
|
|
|
|
|
+ //chromedp.Sleep(1*time.Second),
|
|
|
|
|
+ chromedp.WaitVisible(sel),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("操作：",sel,"失败 error:",err)
|
|
|
|
|
+ //fmt.Println(sel," err:",err)
|
|
|
|
|
+ }
|
|
|
|
|
+ }()
|
|
|
|
|
+
|
|
|
|
|
+ select {
|
|
|
|
|
+ case <-done:
|
|
|
|
|
+ // 操作完成，不需要做任何事情
|
|
|
|
|
+ return nil
|
|
|
|
|
+ case <-ctxTmp.Done():
|
|
|
|
|
+ // 如果 ctxTmp 超时或取消，输出相应信息
|
|
|
|
|
+ err := ctxTmp.Err()
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("操作超时，取消操作")
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func waitVisibleAndClickTimeout(ctx context.Context,sel interface{}) error {
|
|
|
|
|
+ ctxTmp, cancleTmp := context.WithTimeout(ctx, time.Duration(Conf.Timeout)*time.Second)
|
|
|
|
|
+ defer cancleTmp()
|
|
|
|
|
+ done := make(chan struct{})
|
|
|
|
|
+ go func() {
|
|
|
|
|
+ defer close(done)
|
|
|
|
|
+ err := chromedp.Run(ctxTmp,
|
|
|
|
|
+ //chromedp.Sleep(1*time.Second),
|
|
|
|
|
+ chromedp.WaitVisible(sel),
|
|
|
|
|
+ chromedp.Click(sel),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("操作：",sel,"失败 error:",err)
|
|
|
|
|
+ //fmt.Println(sel," err:",err)
|
|
|
|
|
+ }
|
|
|
|
|
+ }()
|
|
|
|
|
+
|
|
|
|
|
+ select {
|
|
|
|
|
+ case <-done:
|
|
|
|
|
+ // 操作完成，不需要做任何事情
|
|
|
|
|
+ return nil
|
|
|
|
|
+ case <-ctxTmp.Done():
|
|
|
|
|
+ // 如果 ctxTmp 超时或取消，输出相应信息
|
|
|
|
|
+ err := ctxTmp.Err()
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("操作超时，取消操作")
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+func getNodeTimeout(ctx context.Context,sel interface{}) ([]*cdp.Node,error) {
|
|
|
|
|
+ var nodes []*cdp.Node
|
|
|
|
|
+ ctxTmp, cancleTmp := context.WithTimeout(ctx, time.Duration(Conf.Timeout)*time.Second)
|
|
|
|
|
+ defer cancleTmp()
|
|
|
|
|
+ done := make(chan struct{})
|
|
|
|
|
+ go func() {
|
|
|
|
|
+ defer close(done)
|
|
|
|
|
+ err := chromedp.Run(ctxTmp,
|
|
|
|
|
+ chromedp.Nodes(sel, &nodes),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("操作：",sel,"失败 error:",err)
|
|
|
|
|
+ //fmt.Println(sel," err:",err)
|
|
|
|
|
+ }
|
|
|
|
|
+ }()
|
|
|
|
|
+
|
|
|
|
|
+ select {
|
|
|
|
|
+ case <-done:
|
|
|
|
|
+ // 操作完成，不需要做任何事情
|
|
|
|
|
+ return nodes,nil
|
|
|
|
|
+ case <-ctxTmp.Done():
|
|
|
|
|
+ // 如果 ctxTmp 超时或取消，输出相应信息
|
|
|
|
|
+ err := ctxTmp.Err()
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("操作超时，取消操作")
|
|
|
|
|
+ return nodes,err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return nodes,nil
|
|
|
|
|
+
|
|
|
|
|
+}
|
|
|
|
|
+func autoLogin(ctx context.Context,userName string) error {
|
|
|
|
|
+ var buf []byte
|
|
|
|
|
+ err := chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Screenshot("body", &buf,chromedp.NodeVisible, chromedp.ByQuery),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("截图失败:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ code := codeGetNew(buf,userName)
|
|
|
|
|
+ //fmt.Println("识别验证码位置:", code)
|
|
|
|
|
+ codeList := strings.Split(code, "|")
|
|
|
|
|
+ if len(codeList) <=2{
|
|
|
|
|
+ return fmt.Errorf("识别异常，小于等于两个验证码")
|
|
|
|
|
+ }
|
|
|
|
|
+ fmt.Println("识别成功")
|
|
|
|
|
+
|
|
|
|
|
+ for _, v := range codeList {
|
|
|
|
|
+ codeX := strings.Split(v, ",")
|
|
|
|
|
+ if len(codeX) < 2 {
|
|
|
|
|
+ return fmt.Errorf("识别异常")
|
|
|
|
|
+ }
|
|
|
|
|
+ x, _ := strconv.Atoi(codeX[0])
|
|
|
|
|
+ y, _ := strconv.Atoi(codeX[1])
|
|
|
|
|
+ //fmt.Println("点击:",float64(x),float64(y))
|
|
|
|
|
+ err = chromedp.Run(ctx, chromedp.MouseClickXY(float64(x),float64(y),chromedp.ButtonLeft))
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("点击错误:", err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ time.Sleep(1 * time.Second)
|
|
|
|
|
+ }
|
|
|
|
|
+ // 执行登录操作
|
|
|
|
|
+ //fmt.Println("点击确定")
|
|
|
|
|
+ err = chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Click(`/html/body/div[4]/div[1]/div[1]/div[2]/div/div/div[2]`),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("点击确定按钮失败:",err )
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+func run(name,userName ,password string,answerMap map[string]AnswerInfo,handleCourse string) error {
|
|
|
|
|
+ userName = strings.TrimSpace(userName)
|
|
|
|
|
+ password = strings.TrimSpace(password)
|
|
|
|
|
+ // 设置Chrome选项
|
|
|
|
|
+ opts := []chromedp.ExecAllocatorOption{
|
|
|
|
|
+ chromedp.Flag("disable-blink-features", "AutomationControlled"),
|
|
|
|
|
+ chromedp.Flag("exclude-switches", "enable-automation"),
|
|
|
|
|
+ chromedp.Flag("disable-popup-blocking", true),
|
|
|
|
|
+ chromedp.Flag("start-maximized", true),
|
|
|
|
|
+ chromedp.Flag("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"),
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // 初始化Chrome实例
|
|
|
|
|
+ allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...)
|
|
|
|
|
+ defer allocCancel()
|
|
|
|
|
+
|
|
|
|
|
+ var err error
|
|
|
|
|
+ ctx, cancel := chromedp.NewContext(allocCtx)
|
|
|
|
|
+ defer cancel()
|
|
|
|
|
+
|
|
|
|
|
+ go func() {
|
|
|
|
|
+ <-ctx.Done()
|
|
|
|
|
+ err := ctx.Err()
|
|
|
|
|
+ if err == context.Canceled {
|
|
|
|
|
+ fmt.Println("浏览器已手动关闭")
|
|
|
|
|
+ } else {
|
|
|
|
|
+ fmt.Println("上下文发生错误:", err)
|
|
|
|
|
+ }
|
|
|
|
|
+ }()
|
|
|
|
|
+
|
|
|
|
|
+ // 执行登录操作
|
|
|
|
|
+ //fmt.Println(name,"登录")
|
|
|
|
|
+ err = chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Navigate("https://iam.pt.ouchn.cn/am/UI/Login?realm=%2F&service=initService&goto=https%3A%2F%2Fiam.pt.ouchn.cn%2Fam%2Foauth2%2Fauthorize%3Fservice%3DinitService%26response_type%3Dcode%26client_id%3D345fcbaf076a4f8a%26scope%3Dall%26redirect_uri%3Dhttps%253A%252F%252Fmenhu.pt.ouchn.cn%252Fouchnapp%252Fwap%252Flogin%252Findex%26decision%3DAllow"),
|
|
|
|
|
+ chromedp.SetValue(`//*[@id="loginName"]`, userName),
|
|
|
|
|
+ chromedp.SetValue(`//*[@id="password"]`, password),
|
|
|
|
|
+ chromedp.Sleep(2*time.Second),
|
|
|
|
|
+ chromedp.Click(`//*[@id="form_button"]`),
|
|
|
|
|
+ chromedp.Sleep(1*time.Second),
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("登录失败:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Println("获取验证码")
|
|
|
|
|
+ err = waitVisibleTimeout(ctx,"/html/body/div[4]/div[1]/div[1]/div[2]/div")
|
|
|
|
|
+
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("获取验证码登录失败:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ time.Sleep(1*time.Second)
|
|
|
|
|
+
|
|
|
|
|
+ if !Conf.CodeManual{
|
|
|
|
|
+ for count := 0;count < 3 ;count++{
|
|
|
|
|
+ err = autoLogin(ctx,userName)
|
|
|
|
|
+ if err == nil {
|
|
|
|
|
+ break
|
|
|
|
|
+ }else{
|
|
|
|
|
+ time.Sleep(10*time.Second)
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Printf("等待进入学生主页\n")
|
|
|
|
|
+
|
|
|
|
|
+ err = waitVisibleTimeout(ctx,"//a[text()='个人信息']")
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("进入学生主页失败：",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ cookie := getCookie(ctx)
|
|
|
|
|
+ time.Sleep(1*time.Second)
|
|
|
|
|
+ // 获取课程列表
|
|
|
|
|
+ courseList := []Course{}
|
|
|
|
|
+ for i:=0 ;i<3 ;i++{
|
|
|
|
|
+ fmt.Println("开始获取课程列表")
|
|
|
|
|
+ courseList = getCourceNew(ctx)
|
|
|
|
|
+ if len(courseList) == 0 {
|
|
|
|
|
+ fmt.Println("获取课程失败，课程为空,重新获取")
|
|
|
|
|
+ time.Sleep(10*time.Second)
|
|
|
|
|
+ continue
|
|
|
|
|
+ }else{
|
|
|
|
|
+ break
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ if len(courseList) == 0 {
|
|
|
|
|
+ fmt.Println("获取课程失败，课程为空")
|
|
|
|
|
+ return fmt.Errorf("获取课程失败，课程为空")
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ for _,v := range courseList{
|
|
|
|
|
+ if !strings.Contains(handleCourse,v.Name){
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Printf("*************正在处理课程(%s)***************\n",v.Name)
|
|
|
|
|
+ err = chromedp.Run(ctx,
|
|
|
|
|
+ chromedp.Navigate(v.Url),
|
|
|
|
|
+ //chromedp.WaitVisible(`//*[@id="student-module-menu"]/div[1]/div[1]/div/a`),
|
|
|
|
|
+ )
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("进入课程失败:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ err = waitVisibleTimeout(ctx,`//*[@id="student-module-menu"]/div[1]/div[1]/div/a`)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("进入课程失败:",err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ courseId := getCourseId(v.Url)
|
|
|
|
|
+ if courseId == "" {
|
|
|
|
|
+ fmt.Println("课程id获取失败")
|
|
|
|
|
+ return fmt.Errorf("课程id获取失败")
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ cookie = getCookie(ctx)
|
|
|
|
|
+ modules, err := getCourseModules(courseId,cookie , false)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Printf("获取module信息失败：%s\n", err.Error())
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ lastLine := 0
|
|
|
|
|
+ var writer *excelize.StreamWriter
|
|
|
|
|
+ var file *excelize.File
|
|
|
|
|
+ if Conf.OnlyCrawlerAnswer{
|
|
|
|
|
+ fmt.Printf("*************正在爬取课程(%s)***************\n",v.Name)
|
|
|
|
|
+ file ,writer = createFile(fmt.Sprintf("%s.xlsx",v.Name))
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ for _, module := range modules {
|
|
|
|
|
+ fmt.Printf("***********课程:%s 栏目:%s***********\n", v.Name, module.Name)
|
|
|
|
|
+ moduleInfo, err := GetModuleInfo(courseId, module.ModuleId, cookie)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Printf("获取module（%s）详细信息失败：%s", module.Name, err.Error())
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ time.Sleep(2*time.Second)
|
|
|
|
|
+
|
|
|
|
|
+ if Conf.OnlyCrawlerAnswer{
|
|
|
|
|
+ lastLine, err = crawlerExame(ctx,moduleInfo,module,courseId,writer,lastLine)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ file.Save()
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if Conf.HandleVideo {
|
|
|
|
|
+ err = handleVideo(ctx,moduleInfo,module,courseId,cookie)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if Conf.HandleAnswer{
|
|
|
|
|
+ err = handleExame(ctx,moduleInfo,module,courseId,cookie,answerMap)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ }
|
|
|
|
|
+ //time.Sleep(100*time.Hour)
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ fmt.Println("学生（",name,"）处理完成")
|
|
|
|
|
+ time.Sleep(2*time.Second)
|
|
|
|
|
+
|
|
|
|
|
+ return nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func GetRecord(filename string) map[string]bool {
|
|
|
|
|
+ ret := map[string]bool{}
|
|
|
|
|
+ fi, err := os.Open(filename)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return ret
|
|
|
|
|
+ }
|
|
|
|
|
+ defer fi.Close()
|
|
|
|
|
+
|
|
|
|
|
+ br := bufio.NewReader(fi)
|
|
|
|
|
+ for {
|
|
|
|
|
+ a, _, c := br.ReadLine()
|
|
|
|
|
+ //fmt.Printf("xx:%s\n", a)
|
|
|
|
|
+ if c == io.EOF {
|
|
|
|
|
+ break
|
|
|
|
|
+ }
|
|
|
|
|
+ array := strings.Split(string(a), " ")
|
|
|
|
|
+ if len(array) != 3 {
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ ret[array[0]] = true
|
|
|
|
|
+ }
|
|
|
|
|
+ return ret
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func GetAcounts(filename string) ([][]string, error) {
|
|
|
|
|
+ visited := GetRecord("record.txt")
|
|
|
|
|
+ fi, err := os.Open(filename)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return nil, err
|
|
|
|
|
+ }
|
|
|
|
|
+ defer fi.Close()
|
|
|
|
|
+ ret := [][]string{}
|
|
|
|
|
+ br := bufio.NewReader(fi)
|
|
|
|
|
+ for {
|
|
|
|
|
+ line, err := br.ReadString('\n')
|
|
|
|
|
+ /*a, _, c := br.ReadLine()
|
|
|
|
|
+ if c == io.EOF {
|
|
|
|
|
+ break
|
|
|
|
|
+ }*/
|
|
|
|
|
+ //line := string(a)
|
|
|
|
|
+ //line = FilterANSI(line)
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ break
|
|
|
|
|
+ }
|
|
|
|
|
+ array := strings.Split(line, ";")
|
|
|
|
|
+ fmt.Println(line)
|
|
|
|
|
+ if len(array) != 3 {
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ if visited[array[0]] {
|
|
|
|
|
+ //fmt.Printf("visited:%v\n", array[2])
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ item := []string{array[0], array[1], array[2]}
|
|
|
|
|
+ fmt.Println(array)
|
|
|
|
|
+ ret = append(ret, item)
|
|
|
|
|
+ }
|
|
|
|
|
+ return ret, nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func columnIndexToName(index int) string {
|
|
|
|
|
+ var name string
|
|
|
|
|
+ for index > 0 {
|
|
|
|
|
+ index--
|
|
|
|
|
+ name = string('A'+index%26) + name
|
|
|
|
|
+ index /= 26
|
|
|
|
|
+ }
|
|
|
|
|
+ return name
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+func saveCell(file *excelize.File ,sheetName string,index ,line int,value string) error {
|
|
|
|
|
+ cell := columnIndexToName(index) + strconv.Itoa(line) // 新列的单元格位置
|
|
|
|
|
+ //fmt.Println("CELL VALUE :",cell,value,index,line)
|
|
|
|
|
+ if err := file.SetCellValue(sheetName, cell,value); err != nil {
|
|
|
|
|
+ fmt.Println(err)
|
|
|
|
|
+ return err
|
|
|
|
|
+ }else{
|
|
|
|
|
+ file.SetRowHeight(sheetName, line,13.5)
|
|
|
|
|
+ //file.SetColWidth(sheetName, columnIndexToName(index), columnIndexToName(index), 11.93) // 设置 A 列的宽度为 30
|
|
|
|
|
+ err = file.Save()
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ return err
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return nil
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+func main(){
|
|
|
|
|
+ err := LoadConfig(*configFile)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("读取配置文件出错：",err)
|
|
|
|
|
+ return
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ var answerMap map[string]AnswerInfo
|
|
|
|
|
+ if Conf.HandleAnswer {
|
|
|
|
|
+ fmt.Println("开始收集答案")
|
|
|
|
|
+ answerMap ,err = loadAnswerFromExcel()
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("收集答案出错：",err)
|
|
|
|
|
+ return
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ //fmt.Println("答案：",answerMap)
|
|
|
|
|
+
|
|
|
|
|
+ go func(){
|
|
|
|
|
+ accountFile, err := excelize.OpenFile("account.xlsx")
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ fmt.Println("open file err:", err)
|
|
|
|
|
+ return
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ for index, sheetName := range accountFile.GetSheetList() {
|
|
|
|
|
+ if index > 0 {
|
|
|
|
|
+ break
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ rows, err := accountFile.GetRows(sheetName)
|
|
|
|
|
+ if err != nil {
|
|
|
|
|
+ return
|
|
|
|
|
+ }
|
|
|
|
|
+ for rowLine, row := range rows {
|
|
|
|
|
+ fmt.Println("开始处理学生：",row[2])
|
|
|
|
|
+ rowLen := len(row)
|
|
|
|
|
+ if rowLen < 4 {
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ if rowLen > 4{
|
|
|
|
|
+ if row[4] == "已处理"{
|
|
|
|
|
+ fmt.Println("已完成处理学生：",row[2])
|
|
|
|
|
+ continue
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ err = run(row[2],row[0],row[1],answerMap,row[3])
|
|
|
|
|
+ if err != nil{
|
|
|
|
|
+ fmt.Println("处理学生：",row[2],"出错:",err)
|
|
|
|
|
+ }else{
|
|
|
|
|
+ saveCell(accountFile,sheetName,5,rowLine+1,"已处理")
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ os.Exit(0)
|
|
|
|
|
+ }()
|
|
|
|
|
+
|
|
|
|
|
+ // 捕获信号
|
|
|
|
|
+ sigChan := make(chan os.Signal, 1)
|
|
|
|
|
+ signal.Notify(sigChan, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL)
|
|
|
|
|
+ sigValue := <-sigChan
|
|
|
|
|
+ fmt.Printf("接收到退出信号:%v", sigValue)
|
|
|
|
|
+}
|
jaryhe
il y a 6 mois