merge test

upload/upload.go

-package upload
-
-import (
-	"archive/zip"
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"github.com/aliyun/aliyun-oss-go-sdk/oss"
-	"github.com/astaxie/beego/orm"
-	"github.com/rocketlaunchr/go-pool"
-	"github.com/xxjwxc/gowp/workpool"
-	"image"
-	"image/jpeg"
-	"image/png"
-	"io"
-	"io/ioutil"
-	"log"
-	"mime"
-	"net/http"
-	"net/http/cookiejar"
-	Qurl "net/url"
-	"os"
-	"path"
-	"ppt_server/models"
-	"regexp"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-)
-
-var (
-	WEBURL string = "http://offcn-live-svc"
-	//WEBURL   string = "https://test-live.offcncloud.com"
-	once     sync.Once
-	syncPool pool.Pool
-)
-
-const docUrl string = "http://doc.offcncloud.com/"
-
-func init() {
-	once.Do(func() {
-		log.SetFlags(log.Lshortfile | log.LstdFlags)
-		syncPool = pool.New(10)
-		syncPool.SetFactory(func() interface{} {
-			return &bytes.Buffer{}
-		})
-	})
-}
-
-func Upload(v *models.XyuSmallTempFiles) {
-
-	go Down(v)
-
-}
-
-func Down(v *models.XyuSmallTempFiles) {
-	o := orm.NewOrm()
-
-	url, _ := Qurl.QueryUnescape(v.Url)
-	filename := path.Base(url)
-	filePath := "/data/files/" + filename
-	ext := path.Ext(filename)
-
-	objectName := v.Url[strings.Index(v.Url, "com/")+4:]
-
-	// 文件以及存在就不需要下载
-
-	if _, err := os.Stat(filePath); err != nil {
-		// Get the data
-		resp, err := http.Get(url)
-		if err != nil {
-			UpdateFileError("1", v.Uuid, v.RoomNum, objectName)
-			log.Println(err)
-			return
-		}
-		defer resp.Body.Close()
-		// 创建一个文件用于保存
-		out, err := os.Create(filePath)
-		if err != nil {
-			log.Println(err)
-			UpdateFileError("1", v.Uuid, v.RoomNum, objectName)
-			return
-		}
-		defer out.Close()
-
-		// 然后将响应流和文件流对接起来
-		_, err = io.Copy(out, resp.Body)
-		if err != nil {
-			log.Println(err)
-			UpdateFileError("1", v.Uuid, v.RoomNum, objectName)
-			return
-		}
-	}
-
-	var room models.XyuRoom
-
-	err := o.QueryTable("xyu_room").Filter("room_num", v.RoomNum).One(&room)
-	if err != nil {
-		log.Println(err)
-		UpdateFileError("2", v.Uuid, v.RoomNum, objectName)
-		return
-	}
-	bucket, err := newOssClient()
-	if err != nil {
-		log.Println(err)
-		UpdateFileError("2", v.Uuid, v.RoomNum, objectName)
-		return
-
-	}
-
-	file, err := os.Open(filePath)
-	if err != nil {
-		log.Println(err)
-		UpdateFileError("2", v.Uuid, v.RoomNum, objectName)
-		return
-	}
-	defer file.Close()
-	room_file := make(map[string]interface{})
-	t := time.Now()
-	now := fmt.Sprintf("%02d-%02d-%02d %02d:%02d:%02d",
-		t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
-
-	slice_path := strings.Split(objectName, "/")
-	log.Println(slice_path[2])
-	room_file["room_id"] = room.Id
-	room_file["id"] = t.Unix()
-	room_file["is_courseware"] = "1"
-	room_file["private"] = "1"
-	room_file["nickname"] = v.Nickanem
-	room_file["name"] = v.Name
-	room_file["time"] = now
-	room_file["uuid"] = v.Uuid
-	if v.Type == 1 {
-		room_file["is_material"] = 1
-	} else if v.Type == 2 {
-		room_file["is_courseware"] = 1
-	} else if v.Type == 3 {
-		room_file["is_titlebook"] = 1
-	}
-	contentHash := slice_path[2]
-	room_file["file_name_hash"] = slice_path[3]
-	room_file["hash"] = contentHash
-
-	//获取文件后缀
-	var doc_type string
-	doc_type = "0"
-
-	if strings.Contains(ext, "ppt") || strings.Contains(ext, "pptx") {
-		doc_type = "4"
-	} else if strings.Contains(ext, "doc") || strings.Contains(ext, "docx") {
-		doc_type = "2"
-	} else if strings.Contains(ext, "xls") || strings.Contains(ext, "xlsx") {
-		doc_type = "3"
-	} else if strings.Contains(ext, "pdf") {
-		doc_type = "1"
-	}
-	room_file["doc_type"] = doc_type
-	_ = mime.AddExtensionType(".pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation")
-	room_file["type"] = mime.TypeByExtension(ext)
-
-	fd, err := os.Open(filePath)
-	if err != nil {
-		log.Println(err)
-		UpdateFileError("2", v.Uuid, v.RoomNum, objectName)
-		return
-	}
-	defer fd.Close()
-
-	log.Println("objectName == ", objectName)
-	objectPath := objectName[0:strings.LastIndex(objectName, "/")] + "/"
-	room_file["path"] = objectName
-	err = bucket.PutObject(objectName, fd)
-	if err != nil {
-		log.Println(err)
-		UpdateFileError("2", v.Uuid, v.RoomNum, objectName)
-		return
-	}
-	furl := "https://xiaoyu-live.oss-cn-beijing-internal.aliyuncs.com/" + objectName
-	maps, err := getFileInfo(furl)
-	if err != nil {
-		log.Println(err)
-		UpdateFileError("2", v.Uuid, v.RoomNum, objectName)
-		return
-	}
-	if e, ok := maps["error"]; ok {
-		log.Println(e)
-		UpdateFileError("2", v.Uuid, v.RoomNum, objectName)
-		return
-	}
-	if _, ok := maps["FileSize"]; !ok {
-		log.Println(maps)
-		UpdateFileError("2", v.Uuid, v.RoomNum, objectName)
-		return
-	}
-	if _, ok := maps["FileSize"].(float64); !ok {
-		log.Println(err)
-		UpdateFileError("2", v.Uuid, v.RoomNum, objectName)
-		return
-	}
-	room_file["files_size"] = strconv.Itoa(int(maps["FileSize"].(float64)))
-	link := strings.Replace(v.Url, "xiaoyu-live.oss-cn-beijing-internal.aliyuncs.com", "desktop.offcncloud.com", -1)
-	room_file["link"] = link
-	wordFloat64 := getWords(maps)
-	room_file["page_count"] = wordFloat64
-
-	downUrl := fmt.Sprintf("http://doc.offcncloud.com/?info=1&words=%v&ssl=1&furl=%s",
-		wordFloat64, furl)
-
-	ch := make(chan bool)
-	go func() {
-		err := downloadFile(downUrl, contentHash, func(length, downLen int64) {
-
-		})
-		if err == nil {
-			ch <- true
-		} else {
-			UpdateFileError("3", v.Uuid, v.RoomNum, objectName)
-			return
-		}
-	}()
-	<-ch
-
-	reader, err := zip.OpenReader("/data/files/" + contentHash + ".zip")
-	if err != nil {
-		log.Println("5===", err)
-		UpdateFileError("4", v.Uuid, v.RoomNum, objectName)
-		return
-	}
-	Pool := workpool.New(100)
-	var imageInfos [][]interface{}
-
-	for _, file := range reader.File {
-		Pool.DoWait(func() error {
-			log.Printf("file = %s \n", file.Name)
-			var err error
-			for i := 0; i < 3; i++ {
-				if imageInfo, err := pdfThumbnail(objectPath, file, bucket); err == nil {
-					imageInfos = append(imageInfos, imageInfo)
-					break
-				} else {
-					log.Println("err = ", err)
-					UpdateFileError("4", v.Uuid, v.RoomNum, objectName)
-					return err
-				}
-
-			}
-			return err
-		})
-	}
-	log.Println("imageInfos == ", imageInfos)
-
-	err = Pool.Wait()
-	if err != nil {
-		UpdateFileError("5", v.Uuid, v.RoomNum, objectName)
-		log.Println(err)
-		return
-	}
-
-	defer reader.Close()
-
-	go func() {
-		err := uploadPacked(bucket, filename, contentHash, objectPath,
-			wordFloat64, imageInfos)
-		if err != nil {
-			UpdateFileError("6", v.Uuid, v.RoomNum, objectName)
-			log.Println(err)
-			return
-		}
-	}()
-	//调用服务端接口写入数据库
-	var roomFile models.XyuRoomFiles
-
-	err = o.QueryTable("xyu_room_files").Filter("room_id", room.Id).Filter("name", filename).One(&roomFile)
-	if err == orm.ErrNoRows {
-
-		err = CreateFileDataBase(room_file)
-		if err != nil {
-			log.Println("err==", err)
-			UpdateFileError("2", v.Uuid, v.RoomNum, objectName)
-			return
-		}
-	}
-
-	elapsed := time.Since(t)
-	log.Println("filename == ", filename, " 上传成功, 上传耗时 == ", elapsed)
-	// 删除压缩包和删除文件
-	defer os.Remove(filePath)
-	defer os.Remove("/data/files/" + contentHash + ".zip")
-	return
-
-}
-
-func webToken() (*http.Client, string, error) {
-	jar, err := cookiejar.New(nil)
-	if err != nil {
-		return &http.Client{}, "", err
-	}
-	client := &http.Client{
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
-			return http.ErrUseLastResponse
-		},
-		Jar: jar,
-	}
-
-
-
-	payload := strings.NewReader("name="+os.Getenv("live_name")+"&password=" + os.Getenv("live_pwd"))
-
-	req, err := http.NewRequest("POST", WEBURL+"/web/login", payload)
-	if err != nil {
-		return &http.Client{}, "", err
-	}
-	req.Header.Add("content-type", "multipart/form-data")
-	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-
-	res, err := client.Do(req)
-
-	if err != nil {
-		log.Println("token请求失败", err)
-		return &http.Client{}, "", err
-	}
-
-	bodys, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		log.Println("获取数据失败", err)
-		return &http.Client{}, "", err
-	}
-
-	defer res.Body.Close()
-
-	response := string(bodys)
-
-	map2 := make(map[string]interface{})
-	err = json.Unmarshal([]byte(response), &map2)
-	if err != nil {
-		return &http.Client{}, "", err
-	}
-	log.Println("获取token == ", map2)
-	if _, ok := map2["code"]; !ok {
-	    return &http.Client{}, "",  errors.New("登录失败")
-	}
-	if _, ok := map2["code"].(float64); !ok {
-	    return &http.Client{}, "",  errors.New("登录失败")
-	}
-	if int(map2["code"].(float64)) == 200 {
-		//登录成功获取token
-		t_res, err := client.Get(WEBURL + "/web/admin")
-		if err != nil {
-			log.Println("token无效", err)
-			return &http.Client{}, "", err
-		}
-		defer t_res.Body.Close()
-		body, err := ioutil.ReadAll(t_res.Body)
-		if err != nil {
-			return &http.Client{}, "", err
-		}
-		re := regexp.MustCompile(`<meta name="csrf-token" content="([^"]+)">`)
-		token := re.FindAllStringSubmatch(string(body), -1)[0][1]
-
-		return client, token, nil
-	} else {
-		return &http.Client{}, "", errors.New("登录失败")
-	}
-}
-
-func UpdateFileError(code, uuid, room_num, path string) error {
-	vv := Qurl.Values{}
-	vv.Add("code", code)
-	vv.Add("uuid", uuid)
-	vv.Add("room_num", room_num)
-	vv.Add("path", path)
-	client, token, err := webToken()
-	if err != nil {
-		log.Println("token 获取失败", err)
-		return err
-	}
-	vv.Add("_token", token)
-	body := vv.Encode()
-	payload := strings.NewReader(body)
-	req, err := http.NewRequest("POST", WEBURL+"/web/room_files_error", payload)
-
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-	req.Header.Add("content-type", "multipart/form-data")
-	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-	res, err := client.Do(req)
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-	defer res.Body.Close()
-
-	post_body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-	map2 := make(map[string]interface{})
-	err = json.Unmarshal(post_body, &map2)
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-	log.Println("room_files_err response = ", map2)
-	if _, ok := map2["code"]; !ok {
-	    return errors.New("消息下发失败")
-	}
-	if _, ok := map2["code"].(float64); !ok {
-	    return errors.New("消息下发失败")
-	}
-	
-	if int(map2["code"].(float64)) != 200 {
-		log.Println("消息下发失败")
-		return errors.New("消息下发失败")
-	}
-	return nil
-}
-
-func CreateFileDataBase(room_file map[string]interface{}) error {
-
-	vv := Qurl.Values{}
-
-	for k, v := range room_file {
-		vv.Add(k, fmt.Sprintf("%v", v))
-	}
-	client, token, err := webToken()
-	if err != nil {
-		log.Println("token 获取失败", err)
-		return err
-	}
-
-	vv.Add("_token", token)
-	body := vv.Encode()
-	payload := strings.NewReader(body)
-	req, err := http.NewRequest("POST", WEBURL+"/web/room_files_add", payload)
-
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-	req.Header.Add("content-type", "multipart/form-data")
-	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-	res, err := client.Do(req)
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-	defer res.Body.Close()
-
-	post_body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-	map2 := make(map[string]interface{})
-	err = json.Unmarshal(post_body, &map2)
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-	
-	log.Println("room_files response = ", map2)
-	
-	if _, ok := map2["code"]; !ok {
-	    return errors.New("上传失败")
-	}
-	if _, ok := map2["code"].(float64); !ok {
-	    return  errors.New("上传失败")
-	}
-
-	if int(map2["code"].(float64)) != 200 {
-		return errors.New("上传失败")
-	}
-	return nil
-}
-
-func pdfThumbnail(objectPath string, file *zip.File, bucket *oss.Bucket) ([]interface{}, error) {
-
-	fc, err := file.Open()
-	if err != nil {
-		return nil, err
-	}
-
-	defer fc.Close()
-
-	//item := syncPool.Borrow()
-	//defer item.Return()
-	//item.MarkAsInvalid()
-	//
-	//buf := item.Item.(*bytes.Buffer)
-
-	var img image.Image
-	fileSuffix := path.Ext(file.Name) //获取文件后缀
-	switch fileSuffix {
-	case ".png":
-		img, _ = png.Decode(fc)
-	case ".jpeg":
-		img, _ = jpeg.Decode(fc)
-	default:
-		return nil, errors.New("image ext is not found")
-	}
-
-	//if err = webp.Encode(buf, img, &webp.Options{Lossless: true}); err != nil {
-	//	return nil, err
-	//}
-	//r := bytes.NewReader(buf.Bytes())
-
-	objectPathName := objectPath + "images/"
-
-	var filenameOnly string
-	filenameOnly = strings.TrimRight(file.Name, fileSuffix)
-	filenameOnly = strings.Replace(filenameOnly, "p", "", -1)
-	fileName := fmt.Sprintf("%05s", filenameOnly) + fileSuffix
-
-	info := img.Bounds()
-	var imageInfo []interface{}
-	imageInfo = append(imageInfo, fileName)
-	imageInfo = append(imageInfo, info.Max.X)
-	imageInfo = append(imageInfo, info.Max.Y)
-
-	fc1, err := file.Open()
-	if err != nil {
-		return nil, err
-	}
-	defer fc1.Close()
-	return imageInfo, bucket.PutObject(objectPathName+fileName, fc1)
-
-
-}
-
-func uploadPacked(bucket *oss.Bucket, filename, contentHash,
-	objectPath string, word float64, imageInfos [][]interface{}) error {
-
-	type packed struct {
-		FileName      string          `json:"fileName"`
-		Hash          string          `json:"hash"`
-		ImagesName    []string        `json:"imagesName"`
-		OssImagesPath string          `json:"ossImagesPath"`
-		ImageInfos    [][]interface{} `json:"imageInfos"`
-	}
-	p := packed{
-		FileName:      filename,
-		Hash:          contentHash,
-		OssImagesPath: objectPath + "images/",
-		ImageInfos:    imageInfos,
-	}
-	for i := 1; i <= int(word); i++ {
-		p.ImagesName = append(p.ImagesName, fmt.Sprintf("%05d", i)+".png")
-	}
-
-	log.Println("imageInfos == ", imageInfos)
-	data, err := json.Marshal(p)
-	if err != nil {
-		return err
-	}
-
-	err = bucket.PutObject(objectPath+"packed.json",
-		strings.NewReader(string(data)))
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func getWords(maps map[string]interface{}) float64 {
-	if word, ok := maps["SlideCount"]; ok {
-		return word.(float64)
-	}
-	if word, ok := maps["PageCount"]; ok {
-		return word.(float64)
-	}
-	if word, ok := maps["SheetCount"]; ok {
-		return word.(float64)
-	}
-	return float64(0)
-}
-
-func getFileInfo(ossFile string) (map[string]interface{}, error) {
-	url := docUrl + "?info=0&ssl=1&furl=" + ossFile
-
-	resp, err := http.Get(url)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	b, err := ioutil.ReadAll(resp.Body)
-
-	if err != nil {
-		return nil, err
-	}
-	var maps map[string]interface{}
-
-	err = json.Unmarshal(b, &maps)
-	if err != nil {
-		return nil, err
-	}
-	return maps, nil
-}
-
-func newOssClient() (*oss.Bucket, error) {
-
-	client, err := oss.New("oss-cn-beijing-internal.aliyuncs.com",
-		os.Getenv("s_access_id"), os.Getenv("s_access_secret"))
-	if err != nil {
-		return nil, err
-	}
-	bucket, err := client.Bucket("xiaoyu-live")
-
-	if err != nil {
-		return nil, err
-	}
-	return bucket, nil
-}
-
-func downloadFile(url, contentHash string, fb func(length, downLen int64)) error {
-	var (
-		fsize   int64
-		buf     = make([]byte, 32*1024)
-		written int64
-	)
-	//创建一个http client
-	client := new(http.Client)
-	//get方法获取资源
-	resp, err := client.Get(url)
-	log.Println(url)
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-	//读取服务器返回的文件大小
-	fsize, err = strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 32)
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-	//创建文件
-
-	file, err := os.Create("/data/files/" + contentHash + ".zip")
-	if err != nil {
-		log.Println(err)
-		return err
-	}
-	defer file.Close()
-	if resp.Body == nil {
-		log.Println(err)
-		return errors.New("body is null")
-	}
-	defer resp.Body.Close()
-	//下面是 io.copyBuffer() 的简化版本
-	for {
-		//读取bytes
-		nr, er := resp.Body.Read(buf)
-		if nr > 0 {
-			//写入bytes
-			nw, ew := file.Write(buf[0:nr])
-			//数据长度大于0
-			if nw > 0 {
-				written += int64(nw)
-			}
-			//写入出错
-			if ew != nil {
-				err = ew
-				break
-			}
-			//读取是数据长度不等于写入的数据长度
-			if nr != nw {
-				err = io.ErrShortWrite
-				break
-			}
-		}
-		if er != nil {
-			if er != io.EOF {
-				err = er
-			}
-			break
-		}
-		//没有错误了快使用 callback
-
-		fb(fsize, written)
-	}
-	return err
-}