commit 3cd797cd4abb567d9917e3230dfd1a7cb829eba9 Author: nuknal Date: Wed May 8 17:28:20 2024 +0800 init diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..36e3ce0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +bin +demo diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..417c108 --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +.DEFAULT_GOAL := out +export DATE := $(shell date +%Y%m%d-%H:%M:%S) +export COMPILE_LDFLAGS='-s -w -X "main.BuildDate=${DATE}" ' +out: + env CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o bin/sjy01-preprocessing-darwin-arm64 -ldflags=${COMPILE_LDFLAGS} *.go + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/sjy01-preprocessing -ldflags=${COMPILE_LDFLAGS} *.go + +clean: + rm -rf ./bin/* \ No newline at end of file diff --git a/aos.go b/aos.go new file mode 100644 index 0000000..93dec61 --- /dev/null +++ b/aos.go @@ -0,0 +1,6 @@ +package main + + + +type AOSFrame struct { +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d915a3a --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module starwiz.cn/sjy01/preprocessing + +go 1.21.0 + +require ( + github.com/duke-git/lancet/v2 v2.3.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/exp v0.0.0-20221208152030-732eee02a75a // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3f97e02 --- /dev/null +++ b/go.sum @@ -0,0 +1,14 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/duke-git/lancet/v2 v2.3.0 h1:Ztie0qOnC4QgGYYqmpmQxbxkPcm54kqFXj1bwhiV8zg= +github.com/duke-git/lancet/v2 v2.3.0/go.mod h1:zGa2R4xswg6EG9I6WnyubDbFO/+A/RROxIbXcwryTsc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/exp v0.0.0-20221208152030-732eee02a75a h1:4iLhBPcpqFmylhnkbY3W0ONLUYYkDAW9xMFLfxgsvCw= +golang.org/x/exp v0.0.0-20221208152030-732eee02a75a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..5f6dbdd --- /dev/null +++ b/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var ( + paramsXML string +) + +var rootCmd = &cobra.Command{ + Use: "SJY01Preprocessing", + Short: "Preprocessing tools for SJY01 dataset", + Long: `Preprocessing tools for SJY01 dataset`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("preprocessing tools for SJY01 dataset") + p := Params{ + InputData: "demo/050722.dat", + OutputPath: "demo/output", + TempPath: "demo/temp", + DataId: "050722", + } + // preprocess(&p) + exractImageData(&p) + }, +} + +func init() { + rootCmd.PersistentFlags().StringVarP(¶msXML, "params", "p", "params.xml", "parameters file path") +} + +func main() { + rootCmd.Execute() +} diff --git a/params.go b/params.go new file mode 100644 index 0000000..ddf1768 --- /dev/null +++ b/params.go @@ -0,0 +1,12 @@ +package main + +type Params struct { + InputData string + OutputPath string + Satellite string + DataId string + LogFile string + TempPath string + Report string + Result string +} diff --git a/process.go b/process.go new file mode 100644 index 0000000..851dafa --- /dev/null +++ b/process.go @@ -0,0 +1,179 @@ +package main + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sort" + + "github.com/duke-git/lancet/v2/slice" +) + +const ( + AOSFrameLength = 1024 +) + +var AOSSyncWord = []byte{0x1A, 0xCF, 0xFC, 0x1D} +var ImageFrameHead = 0xE77EE77E + + +func preprocess(params *Params) error { + // extract data from aos frame + fi, err := os.Open(params.InputData) + if err != nil { + log.Println("exract data from", params.InputData, "error:", err.Error()) + return err + } + defer fi.Close() + + name := filepath.Base(params.InputData) + aosDataFile := filepath.Join(params.TempPath, "aos_data_"+name) + foData, err := os.OpenFile(aosDataFile, + os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777) + if err != nil { + fmt.Println("create data err", err.Error()) + return err + } + defer foData.Close() + wData := bufio.NewWriter(foData) + + var validFrameCnt int + var errFrameCnt int + + iBuf := make([]byte, AOSFrameLength) + r := bufio.NewReader(fi) + for { + correct := true + nI, err := r.Read(iBuf) + if err != nil && err != io.EOF { + panic(err) + } + + if nI < AOSFrameLength || err == io.EOF { + break + } + + iTag := iBuf[0:4] + if iTag[0] == 0x1A && iTag[1] == 0xCF && iTag[2] == 0xFC && iTag[3] == 0x1D { + if err := ldpcCheck(iBuf); err != nil { + errFrameCnt++ + correct = false + } else { + validFrameCnt++ + } + } else { + correct = false + errFrameCnt++ + } + + if !correct { + continue + } + + data := iBuf[10:894] + wData.Write(data) + } + + log.Println("valid frame cnt:", validFrameCnt) + log.Println("err frame cnt:", errFrameCnt) + + return nil +} + +func ldpcCheck(frame []byte) error { + return nil +} + +func exractImageData(params *Params) ([]string, error) { + var sData []string + name := filepath.Base(params.InputData) + aosDataFile := filepath.Join(params.TempPath, "aos_data_"+name) + + aosData, err := os.ReadFile(aosDataFile) + if err != nil { + log.Println("read data from", aosDataFile, "error:", err.Error()) + return nil, err + } + + snRange := map[int][]int{} + datSet := map[int][]byte{} + + var i int + for i < len(aosData) { + if i+4 > len(aosData) { + break + } + + if aosData[i] == 0xE7 && aosData[i+1] == 0x7E && aosData[i+2] == 0xE7 && aosData[i+3] == 0x7E { + i += 4 // <- 帧头 + + serialNum := binary.BigEndian.Uint32(aosData[i : i+4]) + i += 4 // <- 流水号 + i += 2 // <- 保留位 + i += 2 // <- 密钥库索引号 + i += 1 // <- 明密标志 + + flag := aosData[i] + i += 1 // <-帧标志 + + fileno := int(aosData[i]) + snRange[fileno] = append(snRange[fileno], int(serialNum)) + i += 1 // <- 文件号 + + i += 1 // <- 保留 + + if i+ImageFrameDataLength > len(aosData) { + log.Println("frame length error") + break + } + + // 只保留非空帧 + if flag != 0x55 { + datSet[fileno] = append(datSet[fileno], aosData[i:i+ImageFrameDataLength]...) + } + + i += ImageFrameDataLength // <- 图像数据 + if i > len(aosData) { + log.Println("frame length error") + break + } + + // fmt.Println("serialNum:", serialNum, "fileno:", fileno) + continue + } + i++ + } + + for k, v := range datSet { + frameCnt := slice.Unique(snRange[k]) + if len(frameCnt) < 4096 { + log.Println("fileno", k, "数据长度不足4096帧") + continue + } + + s01 := filepath.Join(params.TempPath, fmt.Sprintf("%s_S%d.dat", params.DataId, k)) + fo, err := os.OpenFile(s01, + os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777) + if err != nil { + log.Println("create file", s01, "error:", err.Error()) + return nil, err + } + w := bufio.NewWriter(fo) + w.Write(v) + w.Flush() + fo.Close() + sData = append(sData, s01) + } + + for k, v := range snRange { + vv := slice.Unique(v) + sort.Slice(vv, func(i, j int) bool { return vv[i] < vv[j] }) + fmt.Println("fileno:", k, ", sn range:", vv[0], "-", vv[len(vv)-1], "len", len(vv)) + } + + return sData, nil +} diff --git a/trans_frame.go b/trans_frame.go new file mode 100644 index 0000000..2fe8529 --- /dev/null +++ b/trans_frame.go @@ -0,0 +1,32 @@ +package main + +const ImageFrameHeadLength = 16 +const ImageFrameDataLength = 8176 + +// 智能载荷电箱与数传单机之间图像数据传输帧格式 +type TransFrame struct { + Header [4]byte // 帧头 E77EE77E + SNo uint32 // 序列号 大端序 + Reserved0 [2]byte // 保留字节 + KeyIndex uint16 // 密钥库索引号 大端序 + SecretFlag byte // 明密标志 0x55 明传 0xAA 密传 + FrameFlag byte // 帧类型 0x00:智能载荷电箱有效帧 0x55: 智能载荷电箱空帧 + FileNo byte // 文件编号 + Reserved1 byte // 保留字节 0x00 + Data [8176]byte +} + +func (t *TransFrame) Decode(frame []byte) { + if len(frame) < ImageFrameHeadLength+ImageFrameDataLength { + return + } + copy(t.Header[:], frame[:4]) + t.SNo = uint32(frame[4])<<24 | uint32(frame[5])<<16 | uint32(frame[6])<<8 | uint32(frame[7]) + copy(t.Reserved0[:], frame[8:10]) + t.KeyIndex = uint16(frame[10])<<8 | uint16(frame[11]) + t.SecretFlag = frame[12] + t.FrameFlag = frame[13] + t.FileNo = frame[14] + t.Reserved1 = frame[15] + copy(t.Data[:], frame[16:]) +}