diff --git a/.gitignore b/.gitignore index 36e3ce0..49fb415 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ bin demo +log diff --git a/cmd/extract.go b/cmd/extract.go index 40110c3..057f3e6 100644 --- a/cmd/extract.go +++ b/cmd/extract.go @@ -1,20 +1,26 @@ package cmd import ( + "fmt" + "github.com/spf13/cobra" "starwiz.cn/sjy01/preprocessing/extract" ) +var ( + dataId string +) + var extractCmd = &cobra.Command{ Use: "extract", Short: "Extract data from raw data files", Long: `Extract data from raw data files`, Run: func(cmd *cobra.Command, args []string) { params := extract.Params{ - InputData: "demo/051513.dat", + InputData: fmt.Sprintf("demo/%s.dat", dataId), OutputPath: "demo/output", TempPath: "demo/temp", - DataId: "051513", + DataId: dataId, } p := extract.NewExtractor(¶ms) p.ExtractAosData() @@ -27,4 +33,6 @@ var extractCmd = &cobra.Command{ func init() { rootCmd.AddCommand(extractCmd) + + extractCmd.Flags().StringVarP(&dataId, "data id", "d", "051513", "051513") } diff --git a/extract/aux.go b/extract/aux.go index e786a33..8c43b0f 100644 --- a/extract/aux.go +++ b/extract/aux.go @@ -123,14 +123,14 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error { log.Info("seprate aux and img data from", sDataFile) // 数据质量分析 - fimg, _ := os.Create("demo/temp/aux_img.txt") + fimg, _ := os.Create("demo/temp/" + p.params.DataId + "_aux_img.txt") defer fimg.Close() fimg.WriteString("字节数 帧头流水号 文件号 帧头时间 中心辅助数据前4字节(行33-36)\n") var qmap []*Record for i := 0; i < len(sData); { // 解析帧 if i+24 > len(sData) { - log.Info("length of frame head is not engough for frame head") + log.Info("length of original image frame head is not engough: ", len(sData)-i) break } @@ -146,23 +146,24 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error { afh.Decode(sData[i : i+24]) if !afh.IsValidFrmHead { - log.Info("invalid frame head of original raw data", afh.FrmHead, "i =", i) + log.Debugf("[%d] invalid frame head of original raw data %v", i, afh.FrmHead) i += 1 } else { r := &Record{index: i, frmHead: afh} qmap = append(qmap, r) i += 24 - // fmt.Println("sn:", afh.SerialNo, "time:", afh.TimeSec) - // fmt.Println(time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0).String()) - utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36]) - // fmt.Println("utc time of platform aux:", utcTime) - // fmt.Println(time.Unix(int64(utcTime), 0).String()) - // fmt.Println("waveway:", sData[i+36]) t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0) tAux := time.Unix(int64(utcTime), 0) + startAuxLine := tAux.Year() == t.Year() && tAux.Month() == t.Month() && tAux.Day() == t.Day() && + tAux.Hour() == t.Hour() && tAux.Minute() == t.Minute() + + if startAuxLine { + fimg.WriteString("----- AUX Start -----\n") + } + fimg.WriteString(fmt.Sprintf("%d %d %d %s %s\n", i, afh.SerialNo, @@ -170,11 +171,6 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error { t.String(), tAux.String(), )) - - // if len(qmap) > 32 { - // break - // } - } } @@ -182,27 +178,6 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error { return qmap[i].index < qmap[j].index }) - // lastPosOfPkg := 0 - // lastLenOfPkg := 0 - // for i, v := range qmap { - // pkgLen := v.index - lastPosOfPkg - - // if v.index == 4112 { - // fmt.Println("index 4112's frm no", i+1) - // fmt.Println("last frm:", qmap[i-1].index) - // fmt.Println("next frm:", qmap[i+1].index) - // } - - // if pkgLen != 23872 { - // log.Info("index:", v.index, "lastLenOfPkg:", lastLenOfPkg, "package length:", pkgLen) - // } - - // lastLenOfPkg = pkgLen - // lastPosOfPkg = v.index - // } - - return nil - name := filepath.Base(sDataFile) name = strings.TrimRight(name, filepath.Ext(name)) outputDir := p.params.OutputPath @@ -244,6 +219,8 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error { var afh AuxFrameHead dataLen := len(sData) + + mssData := make([][]byte, 4) for i := 0; i < dataLen; { if i+4 > dataLen { logrus.Println("end of data, dataLen:", dataLen, "i:", i) @@ -281,7 +258,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error { // 读取一行数据 sData[i : i+afh.RowLength] if i+afh.RowLength > dataLen { - log.Info("length of row data is not enough for row length:", afh.RowLength) + log.Infof("length of image row data %v is not enough: %v", dataLen-i, afh.RowLength) break } @@ -300,31 +277,48 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error { } if afh.B1 { - wmss.Write(sData[dataIndex : dataIndex+1192]) + // wmss.Write(sData[dataIndex : dataIndex+1192]) + mssData[0] = append(mssData[0], sData[dataIndex:dataIndex+1192]...) dataIndex += 1192 mssEnviHdr.Lines += 1 } if afh.B2 { - wmss.Write(sData[dataIndex : dataIndex+1192]) + // wmss.Write(sData[dataIndex : dataIndex+1192]) + mssData[1] = append(mssData[1], sData[dataIndex:dataIndex+1192]...) dataIndex += 1192 } if afh.B3 { - wmss.Write(sData[dataIndex : dataIndex+1192]) + // wmss.Write(sData[dataIndex : dataIndex+1192]) + mssData[2] = append(mssData[2], sData[dataIndex:dataIndex+1192]...) dataIndex += 1192 } if afh.B4 { - wmss.Write(sData[dataIndex : dataIndex+1192]) + // wmss.Write(sData[dataIndex : dataIndex+1192]) + mssData[3] = append(mssData[3], sData[dataIndex:dataIndex+1192]...) dataIndex += 1192 } i = dataIndex // 完成一行数据解析 } + var bands = 0 + for i := 0; i < 4; i++ { + if len(mssData[i]) > 0 { + log.Println("write mss data of band B", i+1) + _, err := wmss.Write(mssData[i]) + if err != nil { + log.Error("write mss data error:", err.Error()) + } + bands += 1 + } + } + panEnviHdr.Samples = 9520 panEnviHdr.Bands = 1 wpanHdr.Write([]byte(panEnviHdr.String())) + mssEnviHdr.Lines = mssEnviHdr.Lines / 4 // 多光谱波段分别在 4 行中传输 - mssEnviHdr.Bands = 1 - mssEnviHdr.Samples = 2384 * 4 + mssEnviHdr.Samples = 2384 + mssEnviHdr.Bands = bands wmssHdr.Write([]byte(mssEnviHdr.String())) return nil diff --git a/extract/aux_platform.go b/extract/aux_platform.go index 991be3f..05cef91 100644 --- a/extract/aux_platform.go +++ b/extract/aux_platform.go @@ -136,10 +136,10 @@ func (ap *AuxPlatform) Parse(data []byte) error { ap.DataTransH = float64(binary.BigEndian.Uint16(data[213:215])) ap.SS1_UTCTime = binary.BigEndian.Uint32(data[377:381]) - ap.SS1_UTCTimeFrac = float32(binary.BigEndian.Uint16(data[381:384])) * 40.96 + ap.SS1_UTCTimeFrac = float32(uint32(data[381])<<16|uint32(data[382])<<8|uint32(data[383])) * 40.96 ap.SS1_ImgFrmNo = uint32(data[410])<<16 | uint32(data[411])<<8 | uint32(data[412]) ap.SS2_UTCTime = binary.BigEndian.Uint32(data[423:427]) - ap.SS2_UTCTimeFrac = float32(binary.BigEndian.Uint16(data[427:430])) * 40.96 + ap.SS2_UTCTimeFrac = float32(uint32(data[427])<<16|uint32(data[428])<<8|uint32(data[429])) * 40.96 ap.SS1_ImgFrmNo = uint32(data[456])<<16 | uint32(data[457])<<8 | uint32(data[458]) return nil @@ -180,7 +180,6 @@ func (e *Extractor) ParseAuxPlatform(auxfile string) ([]*AuxPlatform, error) { return nil, err } - ap.Print() aps = append(aps, &ap) if i+AuxPlatformFrmSize > len(data) { diff --git a/extract/process.go b/extract/process.go index 03ef016..de0f714 100644 --- a/extract/process.go +++ b/extract/process.go @@ -6,6 +6,7 @@ import ( type Extractor struct { params *Params + Clean bool } func NewExtractor(params *Params) *Extractor { @@ -13,3 +14,10 @@ func NewExtractor(params *Params) *Extractor { os.MkdirAll(params.TempPath, 0755) return &Extractor{params: params} } + +func (e *Extractor) Cleanup() error { + if e.Clean { + os.RemoveAll(e.params.TempPath) + } + return nil +} diff --git a/extract/trans_frame.go b/extract/trans_frame.go index 1923856..dd0c4c1 100644 --- a/extract/trans_frame.go +++ b/extract/trans_frame.go @@ -83,6 +83,11 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) { snRange[fileno] = append(snRange[fileno], tf.SNo) fsn.WriteString(fmt.Sprintf("%d %d %d\n", i, tf.SNo, fileno)) + if tf.SecretFlag == 0xAA { + log.Info("secret frame is not supported") + break + } + // 只保留非空帧 if tf.FrameFlag != 0x55 && fileno != 0 { datSet[fileno] = append(datSet[fileno], tf.Data...) diff --git a/go.mod b/go.mod index be4e1d7..a88e65b 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,27 @@ go 1.21.0 require github.com/k0kubun/pp v3.0.1+incompatible +require ( + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/term v0.20.0 // indirect +) + require ( github.com/duke-git/lancet/v2 v2.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect + github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect + github.com/x-cray/logrus-prefixed-formatter v0.5.2 golang.org/x/exp v0.0.0-20221208152030-732eee02a75a // indirect - golang.org/x/sys v0.1.0 // indirect + golang.org/x/sys v0.20.0 // indirect ) diff --git a/go.sum b/go.sum index 0a3a11a..3632328 100644 --- a/go.sum +++ b/go.sum @@ -7,11 +7,22 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= +github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= +github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -20,17 +31,26 @@ github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyh github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20221208152030-732eee02a75a h1:4iLhBPcpqFmylhnkbY3W0ONLUYYkDAW9xMFLfxgsvCw= golang.org/x/exp v0.0.0-20221208152030-732eee02a75a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index cc03af6..9b4ecd7 100644 --- a/main.go +++ b/main.go @@ -1,46 +1,10 @@ package main import ( - "fmt" - - "github.com/spf13/cobra" "starwiz.cn/sjy01/preprocessing/cmd" - - "starwiz.cn/sjy01/preprocessing/extract" + _ "starwiz.cn/sjy01/preprocessing/xlog" ) -var ( - paramsXML string -) - -var rootCmd = &cobra.Command{ - Use: "SJY01 Preprocessing", - Short: "Preprocessing tools for SJY01 original raw data", - Long: ``, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("preprocessing tools for SJY01 original raw data") - params := extract.Params{ - InputData: "demo/4545.dat", - OutputPath: "demo/output", - TempPath: "demo/temp", - DataId: "004545", - } - p := extract.NewExtractor(¶ms) - // p.ExtractAosData() - // dats, _ := p.ExtractOriginalImageData() - // for _, d := range dats { - // p.SeprateAuxAndImgData(d) - // } - - p.ParseAuxPlatform("demo/temp/004545_S44_AUX1.dat") - - }, -} - -func init() { - rootCmd.PersistentFlags().StringVarP(¶msXML, "params", "p", "params.xml", "parameters file path") -} - func main() { cmd.Execute() } diff --git a/xlog/log.go b/xlog/log.go new file mode 100644 index 0000000..232ef2a --- /dev/null +++ b/xlog/log.go @@ -0,0 +1,80 @@ +package xlog + +import ( + "os" + "time" + + rotatelogs "github.com/lestrrat-go/file-rotatelogs" + "github.com/rifflock/lfshook" + "github.com/sirupsen/logrus" + prefixed "github.com/x-cray/logrus-prefixed-formatter" +) + +type TagHook struct { + Tag string +} + +func NewTagHook(tag string) logrus.Hook { + hook := TagHook{ + Tag: tag, + } + return &hook +} + +func (hook *TagHook) Fire(entry *logrus.Entry) error { + entry.Data["tag"] = hook.Tag + return nil +} + +func (hook *TagHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +var stdFormatter *prefixed.TextFormatter // 命令行输出格式 +var fileFormatter *prefixed.TextFormatter // 文件输出格式 + +func init() { + stdFormatter = &prefixed.TextFormatter{ + FullTimestamp: true, + TimestampFormat: "2006-01-02.15:04:05", + ForceFormatting: true, + ForceColors: true, + DisableColors: false, + } + fileFormatter = &prefixed.TextFormatter{ + FullTimestamp: true, + TimestampFormat: "2006-01-02.15:04:05", + ForceFormatting: true, + ForceColors: false, + DisableColors: true, + } + + configureLogger(logrus.StandardLogger(), "log/app.log", logrus.InfoLevel) +} + +func NewLogger(logfile string) *logrus.Logger { + logger := logrus.New() + configureLogger(logger, logfile, logrus.InfoLevel) + + return logger +} + +func configureLogger(logger *logrus.Logger, logfile string, level logrus.Level) { + logger.SetFormatter(stdFormatter) + logger.SetLevel(logrus.Level(level)) + writer, _ := rotatelogs.New( + logfile+".%Y%m%d", + rotatelogs.WithLinkName(logfile), + rotatelogs.WithMaxAge(time.Duration(30*24)*time.Hour), + rotatelogs.WithRotationTime(time.Duration(24)*time.Hour), + ) + + lfHook := lfshook.NewHook(lfshook.WriterMap{ + logrus.InfoLevel: writer, + logrus.DebugLevel: writer, + logrus.ErrorLevel: writer, + }, fileFormatter) + + logger.SetOutput(os.Stdout) + logger.AddHook(lfHook) +}