lankun/sjy01-preprocessing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

logger

Browse Source
This commit is contained in:
nuknal
2024-05-16 17:33:33 +08:00
parent 7e4efd83a1
commit 58acd444d6
10 changed files with 175 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,2 +1,3 @@
bin
demo
log

12
cmd/extract.go
View File

@@ -1,20 +1,26 @@
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"starwiz.cn/sjy01/preprocessing/extract"
)
var (
dataId string
)
var extractCmd = &cobra.Command{
Use: "extract",
Short: "Extract data from raw data files",
Long: `Extract data from raw data files`,
Run: func(cmd *cobra.Command, args []string) {
params := extract.Params{
InputData: "demo/051513.dat",
InputData: fmt.Sprintf("demo/%s.dat", dataId),
OutputPath: "demo/output",
TempPath: "demo/temp",
DataId: "051513",
DataId: dataId,
}
p := extract.NewExtractor(&params)
p.ExtractAosData()
@@ -27,4 +33,6 @@ var extractCmd = &cobra.Command{
func init() {
rootCmd.AddCommand(extractCmd)
extractCmd.Flags().StringVarP(&dataId, "data id", "d", "051513", "051513")
}

78
extract/aux.go
View File

@@ -123,14 +123,14 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
log.Info("seprate aux and img data from", sDataFile)
// 数据质量分析
fimg, _ := os.Create("demo/temp/aux_img.txt")
fimg, _ := os.Create("demo/temp/" + p.params.DataId + "_aux_img.txt")
defer fimg.Close()
fimg.WriteString("字节数 帧头流水号 文件号 帧头时间 中心辅助数据前4字节(行33-36)\n")
var qmap []*Record
for i := 0; i < len(sData); {
// 解析帧
if i+24 > len(sData) {
log.Info("length of frame head is not engough for frame head")
log.Info("length of original image frame head is not engough: ", len(sData)-i)
break
}
@@ -146,23 +146,24 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
afh.Decode(sData[i : i+24])
if !afh.IsValidFrmHead {
log.Info("invalid frame head of original raw data", afh.FrmHead, "i =", i)
log.Debugf("[%d] invalid frame head of original raw data %v", i, afh.FrmHead)
i += 1
} else {
r := &Record{index: i, frmHead: afh}
qmap = append(qmap, r)
i += 24
// fmt.Println("sn:", afh.SerialNo, "time:", afh.TimeSec)
// fmt.Println(time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0).String())
utcTime := binary.BigEndian.Uint32(sData[i+32 : i+36])
// fmt.Println("utc time of platform aux:", utcTime)
// fmt.Println(time.Unix(int64(utcTime), 0).String())
// fmt.Println("waveway:", sData[i+36])
t := time.Unix(int64(afh.TimeSec+uint32(ReferenceTime2000)), 0)
tAux := time.Unix(int64(utcTime), 0)
startAuxLine := tAux.Year() == t.Year() && tAux.Month() == t.Month() && tAux.Day() == t.Day() &&
tAux.Hour() == t.Hour() && tAux.Minute() == t.Minute()
if startAuxLine {
fimg.WriteString("----- AUX Start -----\n")
}
fimg.WriteString(fmt.Sprintf("%d %d %d %s %s\n",
i,
afh.SerialNo,
@@ -170,11 +171,6 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
t.String(),
tAux.String(),
))
// if len(qmap) > 32 {
// break
// }
}
}
@@ -182,27 +178,6 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
return qmap[i].index < qmap[j].index
})
// lastPosOfPkg := 0
// lastLenOfPkg := 0
// for i, v := range qmap {
// pkgLen := v.index - lastPosOfPkg
// if v.index == 4112 {
// fmt.Println("index 4112's frm no", i+1)
// fmt.Println("last frm:", qmap[i-1].index)
// fmt.Println("next frm:", qmap[i+1].index)
// }
// if pkgLen != 23872 {
// log.Info("index:", v.index, "lastLenOfPkg:", lastLenOfPkg, "package length:", pkgLen)
// }
// lastLenOfPkg = pkgLen
// lastPosOfPkg = v.index
// }
return nil
name := filepath.Base(sDataFile)
name = strings.TrimRight(name, filepath.Ext(name))
outputDir := p.params.OutputPath
@@ -244,6 +219,8 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
var afh AuxFrameHead
dataLen := len(sData)
mssData := make([][]byte, 4)
for i := 0; i < dataLen; {
if i+4 > dataLen {
logrus.Println("end of data, dataLen:", dataLen, "i:", i)
@@ -281,7 +258,7 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
// 读取一行数据 sData[i : i+afh.RowLength]
if i+afh.RowLength > dataLen {
log.Info("length of row data is not enough for row length:", afh.RowLength)
log.Infof("length of image row data %v is not enough: %v", dataLen-i, afh.RowLength)
break
}
@@ -300,31 +277,48 @@ func (p *Extractor) SeprateAuxAndImgData(sDataFile string) error {
}
if afh.B1 {
wmss.Write(sData[dataIndex : dataIndex+1192])
// wmss.Write(sData[dataIndex : dataIndex+1192])
mssData[0] = append(mssData[0], sData[dataIndex:dataIndex+1192]...)
dataIndex += 1192
mssEnviHdr.Lines += 1
}
if afh.B2 {
wmss.Write(sData[dataIndex : dataIndex+1192])
// wmss.Write(sData[dataIndex : dataIndex+1192])
mssData[1] = append(mssData[1], sData[dataIndex:dataIndex+1192]...)
dataIndex += 1192
}
if afh.B3 {
wmss.Write(sData[dataIndex : dataIndex+1192])
// wmss.Write(sData[dataIndex : dataIndex+1192])
mssData[2] = append(mssData[2], sData[dataIndex:dataIndex+1192]...)
dataIndex += 1192
}
if afh.B4 {
wmss.Write(sData[dataIndex : dataIndex+1192])
// wmss.Write(sData[dataIndex : dataIndex+1192])
mssData[3] = append(mssData[3], sData[dataIndex:dataIndex+1192]...)
dataIndex += 1192
}
i = dataIndex // 完成一行数据解析
}
var bands = 0
for i := 0; i < 4; i++ {
if len(mssData[i]) > 0 {
log.Println("write mss data of band B", i+1)
_, err := wmss.Write(mssData[i])
if err != nil {
log.Error("write mss data error:", err.Error())
}
bands += 1
}
}
panEnviHdr.Samples = 9520
panEnviHdr.Bands = 1
wpanHdr.Write([]byte(panEnviHdr.String()))
mssEnviHdr.Lines = mssEnviHdr.Lines / 4 // 多光谱波段分别在 4 行中传输
mssEnviHdr.Bands = 1
mssEnviHdr.Samples = 2384 * 4
mssEnviHdr.Samples = 2384
mssEnviHdr.Bands = bands
wmssHdr.Write([]byte(mssEnviHdr.String()))
return nil

5
extract/aux_platform.go
View File

@@ -136,10 +136,10 @@ func (ap *AuxPlatform) Parse(data []byte) error {
ap.DataTransH = float64(binary.BigEndian.Uint16(data[213:215]))
ap.SS1_UTCTime = binary.BigEndian.Uint32(data[377:381])
ap.SS1_UTCTimeFrac = float32(binary.BigEndian.Uint16(data[381:384])) * 40.96
ap.SS1_UTCTimeFrac = float32(uint32(data[381])<<16|uint32(data[382])<<8|uint32(data[383])) * 40.96
ap.SS1_ImgFrmNo = uint32(data[410])<<16 | uint32(data[411])<<8 | uint32(data[412])
ap.SS2_UTCTime = binary.BigEndian.Uint32(data[423:427])
ap.SS2_UTCTimeFrac = float32(binary.BigEndian.Uint16(data[427:430])) * 40.96
ap.SS2_UTCTimeFrac = float32(uint32(data[427])<<16|uint32(data[428])<<8|uint32(data[429])) * 40.96
ap.SS1_ImgFrmNo = uint32(data[456])<<16 | uint32(data[457])<<8 | uint32(data[458])
return nil
@@ -180,7 +180,6 @@ func (e *Extractor) ParseAuxPlatform(auxfile string) ([]*AuxPlatform, error) {
return nil, err
}
ap.Print()
aps = append(aps, &ap)
if i+AuxPlatformFrmSize > len(data) {

8
extract/process.go
View File

@@ -6,6 +6,7 @@ import (
type Extractor struct {
params *Params
Clean bool
}
func NewExtractor(params *Params) *Extractor {
@@ -13,3 +14,10 @@ func NewExtractor(params *Params) *Extractor {
os.MkdirAll(params.TempPath, 0755)
return &Extractor{params: params}
}
func (e *Extractor) Cleanup() error {
if e.Clean {
os.RemoveAll(e.params.TempPath)
}
return nil
}

5
extract/trans_frame.go
View File

@@ -83,6 +83,11 @@ func (p *Extractor) ExtractOriginalImageData() ([]string, error) {
snRange[fileno] = append(snRange[fileno], tf.SNo)
fsn.WriteString(fmt.Sprintf("%d %d %d\n", i, tf.SNo, fileno))
if tf.SecretFlag == 0xAA {
log.Info("secret frame is not supported")
break
}
// 只保留非空帧
if tf.FrameFlag != 0x55 && fileno != 0 {
datSet[fileno] = append(datSet[fileno], tf.Data...)

13
go.mod
View File

@@ -4,16 +4,27 @@ go 1.21.0
require github.com/k0kubun/pp v3.0.1+incompatible
require (
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/term v0.20.0 // indirect
)
require (
github.com/duke-git/lancet/v2 v2.3.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect
github.com/lestrrat-go/strftime v1.0.6 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spf13/cobra v1.8.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/x-cray/logrus-prefixed-formatter v0.5.2
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/sys v0.20.0 // indirect
)

20
go.sum
View File

@@ -7,11 +7,22 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40=
github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is=
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4=
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA=
github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ=
github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
@@ -20,17 +31,26 @@ github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyh
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=
github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a h1:4iLhBPcpqFmylhnkbY3W0ONLUYYkDAW9xMFLfxgsvCw=
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

38
main.go
View File

@@ -1,46 +1,10 @@
package main
import (
"fmt"
"github.com/spf13/cobra"
"starwiz.cn/sjy01/preprocessing/cmd"
"starwiz.cn/sjy01/preprocessing/extract"
_ "starwiz.cn/sjy01/preprocessing/xlog"
)
var (
paramsXML string
)
var rootCmd = &cobra.Command{
Use: "SJY01 Preprocessing",
Short: "Preprocessing tools for SJY01 original raw data",
Long: ``,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("preprocessing tools for SJY01 original raw data")
params := extract.Params{
InputData: "demo/4545.dat",
OutputPath: "demo/output",
TempPath: "demo/temp",
DataId: "004545",
}
p := extract.NewExtractor(&params)
// p.ExtractAosData()
// dats, _ := p.ExtractOriginalImageData()
// for _, d := range dats {
// p.SeprateAuxAndImgData(d)
// }
p.ParseAuxPlatform("demo/temp/004545_S44_AUX1.dat")
},
}
func init() {
rootCmd.PersistentFlags().StringVarP(&paramsXML, "params", "p", "params.xml", "parameters file path")
}
func main() {
cmd.Execute()
}

80
xlog/log.go Normal file
View File

@@ -0,0 +1,80 @@
package xlog
import (
"os"
"time"
rotatelogs "github.com/lestrrat-go/file-rotatelogs"
"github.com/rifflock/lfshook"
"github.com/sirupsen/logrus"
prefixed "github.com/x-cray/logrus-prefixed-formatter"
)
type TagHook struct {
Tag string
}
func NewTagHook(tag string) logrus.Hook {
hook := TagHook{
Tag: tag,
}
return &hook
}
func (hook *TagHook) Fire(entry *logrus.Entry) error {
entry.Data["tag"] = hook.Tag
return nil
}
func (hook *TagHook) Levels() []logrus.Level {
return logrus.AllLevels
}
var stdFormatter *prefixed.TextFormatter // 命令行输出格式
var fileFormatter *prefixed.TextFormatter // 文件输出格式
func init() {
stdFormatter = &prefixed.TextFormatter{
FullTimestamp: true,
TimestampFormat: "2006-01-02.15:04:05",
ForceFormatting: true,
ForceColors: true,
DisableColors: false,
}
fileFormatter = &prefixed.TextFormatter{
FullTimestamp: true,
TimestampFormat: "2006-01-02.15:04:05",
ForceFormatting: true,
ForceColors: false,
DisableColors: true,
}
configureLogger(logrus.StandardLogger(), "log/app.log", logrus.InfoLevel)
}
func NewLogger(logfile string) *logrus.Logger {
logger := logrus.New()
configureLogger(logger, logfile, logrus.InfoLevel)
return logger
}
func configureLogger(logger *logrus.Logger, logfile string, level logrus.Level) {
logger.SetFormatter(stdFormatter)
logger.SetLevel(logrus.Level(level))
writer, _ := rotatelogs.New(
logfile+".%Y%m%d",
rotatelogs.WithLinkName(logfile),
rotatelogs.WithMaxAge(time.Duration(30*24)*time.Hour),
rotatelogs.WithRotationTime(time.Duration(24)*time.Hour),
)
lfHook := lfshook.NewHook(lfshook.WriterMap{
logrus.InfoLevel: writer,
logrus.DebugLevel: writer,
logrus.ErrorLevel: writer,
}, fileFormatter)
logger.SetOutput(os.Stdout)
logger.AddHook(lfHook)
}