From 3cd797cd4abb567d9917e3230dfd1a7cb829eba9 Mon Sep 17 00:00:00 2001 From: nuknal Date: Wed, 8 May 2024 17:28:20 +0800 Subject: [PATCH] init --- .gitignore | 2 + Makefile | 9 +++ aos.go | 6 ++ go.mod | 11 +++ go.sum | 14 ++++ main.go | 36 ++++++++++ params.go | 12 ++++ process.go | 179 +++++++++++++++++++++++++++++++++++++++++++++++++ trans_frame.go | 32 +++++++++ 9 files changed, 301 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 aos.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 params.go create mode 100644 process.go create mode 100644 trans_frame.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..36e3ce0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +bin +demo diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..417c108 --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +.DEFAULT_GOAL := out +export DATE := $(shell date +%Y%m%d-%H:%M:%S) +export COMPILE_LDFLAGS='-s -w -X "main.BuildDate=${DATE}" ' +out: + env CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o bin/sjy01-preprocessing-darwin-arm64 -ldflags=${COMPILE_LDFLAGS} *.go + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/sjy01-preprocessing -ldflags=${COMPILE_LDFLAGS} *.go + +clean: + rm -rf ./bin/* \ No newline at end of file diff --git a/aos.go b/aos.go new file mode 100644 index 0000000..93dec61 --- /dev/null +++ b/aos.go @@ -0,0 +1,6 @@ +package main + + + +type AOSFrame struct { +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d915a3a --- /dev/null +++ b/go.mod @@ -0,0 +1,11 @@ +module starwiz.cn/sjy01/preprocessing + +go 1.21.0 + +require ( + github.com/duke-git/lancet/v2 v2.3.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/exp v0.0.0-20221208152030-732eee02a75a // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3f97e02 --- /dev/null +++ b/go.sum @@ -0,0 +1,14 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/duke-git/lancet/v2 v2.3.0 h1:Ztie0qOnC4QgGYYqmpmQxbxkPcm54kqFXj1bwhiV8zg= +github.com/duke-git/lancet/v2 v2.3.0/go.mod h1:zGa2R4xswg6EG9I6WnyubDbFO/+A/RROxIbXcwryTsc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/exp v0.0.0-20221208152030-732eee02a75a h1:4iLhBPcpqFmylhnkbY3W0ONLUYYkDAW9xMFLfxgsvCw= +golang.org/x/exp v0.0.0-20221208152030-732eee02a75a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..5f6dbdd --- /dev/null +++ b/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var ( + paramsXML string +) + +var rootCmd = &cobra.Command{ + Use: "SJY01Preprocessing", + Short: "Preprocessing tools for SJY01 dataset", + Long: `Preprocessing tools for SJY01 dataset`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("preprocessing tools for SJY01 dataset") + p := Params{ + InputData: "demo/050722.dat", + OutputPath: "demo/output", + TempPath: "demo/temp", + DataId: "050722", + } + // preprocess(&p) + exractImageData(&p) + }, +} + +func init() { + rootCmd.PersistentFlags().StringVarP(¶msXML, "params", "p", "params.xml", "parameters file path") +} + +func main() { + rootCmd.Execute() +} diff --git a/params.go b/params.go new file mode 100644 index 0000000..ddf1768 --- /dev/null +++ b/params.go @@ -0,0 +1,12 @@ +package main + +type Params struct { + InputData string + OutputPath string + Satellite string + DataId string + LogFile string + TempPath string + Report string + Result string +} diff --git a/process.go b/process.go new file mode 100644 index 0000000..851dafa --- /dev/null +++ b/process.go @@ -0,0 +1,179 @@ +package main + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sort" + + "github.com/duke-git/lancet/v2/slice" +) + +const ( + AOSFrameLength = 1024 +) + +var AOSSyncWord = []byte{0x1A, 0xCF, 0xFC, 0x1D} +var ImageFrameHead = 0xE77EE77E + + +func preprocess(params *Params) error { + // extract data from aos frame + fi, err := os.Open(params.InputData) + if err != nil { + log.Println("exract data from", params.InputData, "error:", err.Error()) + return err + } + defer fi.Close() + + name := filepath.Base(params.InputData) + aosDataFile := filepath.Join(params.TempPath, "aos_data_"+name) + foData, err := os.OpenFile(aosDataFile, + os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777) + if err != nil { + fmt.Println("create data err", err.Error()) + return err + } + defer foData.Close() + wData := bufio.NewWriter(foData) + + var validFrameCnt int + var errFrameCnt int + + iBuf := make([]byte, AOSFrameLength) + r := bufio.NewReader(fi) + for { + correct := true + nI, err := r.Read(iBuf) + if err != nil && err != io.EOF { + panic(err) + } + + if nI < AOSFrameLength || err == io.EOF { + break + } + + iTag := iBuf[0:4] + if iTag[0] == 0x1A && iTag[1] == 0xCF && iTag[2] == 0xFC && iTag[3] == 0x1D { + if err := ldpcCheck(iBuf); err != nil { + errFrameCnt++ + correct = false + } else { + validFrameCnt++ + } + } else { + correct = false + errFrameCnt++ + } + + if !correct { + continue + } + + data := iBuf[10:894] + wData.Write(data) + } + + log.Println("valid frame cnt:", validFrameCnt) + log.Println("err frame cnt:", errFrameCnt) + + return nil +} + +func ldpcCheck(frame []byte) error { + return nil +} + +func exractImageData(params *Params) ([]string, error) { + var sData []string + name := filepath.Base(params.InputData) + aosDataFile := filepath.Join(params.TempPath, "aos_data_"+name) + + aosData, err := os.ReadFile(aosDataFile) + if err != nil { + log.Println("read data from", aosDataFile, "error:", err.Error()) + return nil, err + } + + snRange := map[int][]int{} + datSet := map[int][]byte{} + + var i int + for i < len(aosData) { + if i+4 > len(aosData) { + break + } + + if aosData[i] == 0xE7 && aosData[i+1] == 0x7E && aosData[i+2] == 0xE7 && aosData[i+3] == 0x7E { + i += 4 // <- 帧头 + + serialNum := binary.BigEndian.Uint32(aosData[i : i+4]) + i += 4 // <- 流水号 + i += 2 // <- 保留位 + i += 2 // <- 密钥库索引号 + i += 1 // <- 明密标志 + + flag := aosData[i] + i += 1 // <-帧标志 + + fileno := int(aosData[i]) + snRange[fileno] = append(snRange[fileno], int(serialNum)) + i += 1 // <- 文件号 + + i += 1 // <- 保留 + + if i+ImageFrameDataLength > len(aosData) { + log.Println("frame length error") + break + } + + // 只保留非空帧 + if flag != 0x55 { + datSet[fileno] = append(datSet[fileno], aosData[i:i+ImageFrameDataLength]...) + } + + i += ImageFrameDataLength // <- 图像数据 + if i > len(aosData) { + log.Println("frame length error") + break + } + + // fmt.Println("serialNum:", serialNum, "fileno:", fileno) + continue + } + i++ + } + + for k, v := range datSet { + frameCnt := slice.Unique(snRange[k]) + if len(frameCnt) < 4096 { + log.Println("fileno", k, "数据长度不足4096帧") + continue + } + + s01 := filepath.Join(params.TempPath, fmt.Sprintf("%s_S%d.dat", params.DataId, k)) + fo, err := os.OpenFile(s01, + os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777) + if err != nil { + log.Println("create file", s01, "error:", err.Error()) + return nil, err + } + w := bufio.NewWriter(fo) + w.Write(v) + w.Flush() + fo.Close() + sData = append(sData, s01) + } + + for k, v := range snRange { + vv := slice.Unique(v) + sort.Slice(vv, func(i, j int) bool { return vv[i] < vv[j] }) + fmt.Println("fileno:", k, ", sn range:", vv[0], "-", vv[len(vv)-1], "len", len(vv)) + } + + return sData, nil +} diff --git a/trans_frame.go b/trans_frame.go new file mode 100644 index 0000000..2fe8529 --- /dev/null +++ b/trans_frame.go @@ -0,0 +1,32 @@ +package main + +const ImageFrameHeadLength = 16 +const ImageFrameDataLength = 8176 + +// 智能载荷电箱与数传单机之间图像数据传输帧格式 +type TransFrame struct { + Header [4]byte // 帧头 E77EE77E + SNo uint32 // 序列号 大端序 + Reserved0 [2]byte // 保留字节 + KeyIndex uint16 // 密钥库索引号 大端序 + SecretFlag byte // 明密标志 0x55 明传 0xAA 密传 + FrameFlag byte // 帧类型 0x00:智能载荷电箱有效帧 0x55: 智能载荷电箱空帧 + FileNo byte // 文件编号 + Reserved1 byte // 保留字节 0x00 + Data [8176]byte +} + +func (t *TransFrame) Decode(frame []byte) { + if len(frame) < ImageFrameHeadLength+ImageFrameDataLength { + return + } + copy(t.Header[:], frame[:4]) + t.SNo = uint32(frame[4])<<24 | uint32(frame[5])<<16 | uint32(frame[6])<<8 | uint32(frame[7]) + copy(t.Reserved0[:], frame[8:10]) + t.KeyIndex = uint16(frame[10])<<8 | uint16(frame[11]) + t.SecretFlag = frame[12] + t.FrameFlag = frame[13] + t.FileNo = frame[14] + t.Reserved1 = frame[15] + copy(t.Data[:], frame[16:]) +}