init

.gitignore

@@ -0,0 +1,2 @@
+bin
+demo

Makefile

@@ -0,0 +1,9 @@
+.DEFAULT_GOAL := out
+export DATE := $(shell date +%Y%m%d-%H:%M:%S)
+export COMPILE_LDFLAGS='-s -w -X "main.BuildDate=${DATE}" '
+out:
+	env CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o bin/sjy01-preprocessing-darwin-arm64 -ldflags=${COMPILE_LDFLAGS} *.go 
+	env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/sjy01-preprocessing -ldflags=${COMPILE_LDFLAGS} *.go
+
+clean:
+	rm -rf ./bin/*

aos.go

@@ -0,0 +1,6 @@
+package main
+
+
+
+type AOSFrame struct {
+}

go.mod

@@ -0,0 +1,11 @@
+module starwiz.cn/sjy01/preprocessing
+
+go 1.21.0
+
+require (
+	github.com/duke-git/lancet/v2 v2.3.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/exp v0.0.0-20221208152030-732eee02a75a // indirect
+)

go.sum

@@ -0,0 +1,14 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/duke-git/lancet/v2 v2.3.0 h1:Ztie0qOnC4QgGYYqmpmQxbxkPcm54kqFXj1bwhiV8zg=
+github.com/duke-git/lancet/v2 v2.3.0/go.mod h1:zGa2R4xswg6EG9I6WnyubDbFO/+A/RROxIbXcwryTsc=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+golang.org/x/exp v0.0.0-20221208152030-732eee02a75a h1:4iLhBPcpqFmylhnkbY3W0ONLUYYkDAW9xMFLfxgsvCw=
+golang.org/x/exp v0.0.0-20221208152030-732eee02a75a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	paramsXML string
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "SJY01Preprocessing",
+	Short: "Preprocessing tools for SJY01 dataset",
+	Long:  `Preprocessing tools for SJY01 dataset`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("preprocessing tools for SJY01 dataset")
+		p := Params{
+			InputData:  "demo/050722.dat",
+			OutputPath: "demo/output",
+			TempPath:   "demo/temp",
+			DataId:     "050722",
+		}
+		// preprocess(&p)
+		exractImageData(&p)
+	},
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringVarP(&paramsXML, "params", "p", "params.xml", "parameters file path")
+}
+
+func main() {
+	rootCmd.Execute()
+}

params.go

@@ -0,0 +1,12 @@
+package main
+
+type Params struct {
+	InputData  string
+	OutputPath string
+	Satellite  string
+	DataId     string
+	LogFile    string
+	TempPath   string
+	Report     string
+	Result     string
+}

process.go

@@ -0,0 +1,179 @@
+package main
+
+import (
+	"bufio"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/duke-git/lancet/v2/slice"
+)
+
+const (
+	AOSFrameLength = 1024
+)
+
+var AOSSyncWord = []byte{0x1A, 0xCF, 0xFC, 0x1D}
+var ImageFrameHead = 0xE77EE77E
+
+
+func preprocess(params *Params) error {
+	// extract data from aos frame
+	fi, err := os.Open(params.InputData)
+	if err != nil {
+		log.Println("exract data from", params.InputData, "error:", err.Error())
+		return err
+	}
+	defer fi.Close()
+
+	name := filepath.Base(params.InputData)
+	aosDataFile := filepath.Join(params.TempPath, "aos_data_"+name)
+	foData, err := os.OpenFile(aosDataFile,
+		os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777)
+	if err != nil {
+		fmt.Println("create data err", err.Error())
+		return err
+	}
+	defer foData.Close()
+	wData := bufio.NewWriter(foData)
+
+	var validFrameCnt int
+	var errFrameCnt int
+
+	iBuf := make([]byte, AOSFrameLength)
+	r := bufio.NewReader(fi)
+	for {
+		correct := true
+		nI, err := r.Read(iBuf)
+		if err != nil && err != io.EOF {
+			panic(err)
+		}
+
+		if nI < AOSFrameLength || err == io.EOF {
+			break
+		}
+
+		iTag := iBuf[0:4]
+		if iTag[0] == 0x1A && iTag[1] == 0xCF && iTag[2] == 0xFC && iTag[3] == 0x1D {
+			if err := ldpcCheck(iBuf); err != nil {
+				errFrameCnt++
+				correct = false
+			} else {
+				validFrameCnt++
+			}
+		} else {
+			correct = false
+			errFrameCnt++
+		}
+
+		if !correct {
+			continue
+		}
+
+		data := iBuf[10:894]
+		wData.Write(data)
+	}
+
+	log.Println("valid frame cnt:", validFrameCnt)
+	log.Println("err frame cnt:", errFrameCnt)
+
+	return nil
+}
+
+func ldpcCheck(frame []byte) error {
+	return nil
+}
+
+func exractImageData(params *Params) ([]string, error) {
+	var sData []string
+	name := filepath.Base(params.InputData)
+	aosDataFile := filepath.Join(params.TempPath, "aos_data_"+name)
+
+	aosData, err := os.ReadFile(aosDataFile)
+	if err != nil {
+		log.Println("read data from", aosDataFile, "error:", err.Error())
+		return nil, err
+	}
+
+	snRange := map[int][]int{}
+	datSet := map[int][]byte{}
+
+	var i int
+	for i < len(aosData) {
+		if i+4 > len(aosData) {
+			break
+		}
+
+		if aosData[i] == 0xE7 && aosData[i+1] == 0x7E && aosData[i+2] == 0xE7 && aosData[i+3] == 0x7E {
+			i += 4 // <- 帧头
+
+			serialNum := binary.BigEndian.Uint32(aosData[i : i+4])
+			i += 4 // <- 流水号
+			i += 2 // <- 保留位
+			i += 2 // <- 密钥库索引号
+			i += 1 // <- 明密标志
+
+			flag := aosData[i]
+			i += 1 // <-帧标志
+
+			fileno := int(aosData[i])
+			snRange[fileno] = append(snRange[fileno], int(serialNum))
+			i += 1 // <- 文件号
+
+			i += 1 // <- 保留
+
+			if i+ImageFrameDataLength > len(aosData) {
+				log.Println("frame length error")
+				break
+			}
+
+			// 只保留非空帧
+			if flag != 0x55 {
+				datSet[fileno] = append(datSet[fileno], aosData[i:i+ImageFrameDataLength]...)
+			}
+
+			i += ImageFrameDataLength // <- 图像数据
+			if i > len(aosData) {
+				log.Println("frame length error")
+				break
+			}
+
+			// fmt.Println("serialNum:", serialNum, "fileno:", fileno)
+			continue
+		}
+		i++
+	}
+
+	for k, v := range datSet {
+		frameCnt := slice.Unique(snRange[k])
+		if len(frameCnt) < 4096 {
+			log.Println("fileno", k, "数据长度不足4096帧")
+			continue
+		}
+
+		s01 := filepath.Join(params.TempPath, fmt.Sprintf("%s_S%d.dat", params.DataId, k))
+		fo, err := os.OpenFile(s01,
+			os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777)
+		if err != nil {
+			log.Println("create file", s01, "error:", err.Error())
+			return nil, err
+		}
+		w := bufio.NewWriter(fo)
+		w.Write(v)
+		w.Flush()
+		fo.Close()
+		sData = append(sData, s01)
+	}
+
+	for k, v := range snRange {
+		vv := slice.Unique(v)
+		sort.Slice(vv, func(i, j int) bool { return vv[i] < vv[j] })
+		fmt.Println("fileno:", k, ", sn range:", vv[0], "-", vv[len(vv)-1], "len", len(vv))
+	}
+
+	return sData, nil
+}

trans_frame.go

@@ -0,0 +1,32 @@
+package main
+
+const ImageFrameHeadLength = 16
+const ImageFrameDataLength = 8176
+
+// 智能载荷电箱与数传单机之间图像数据传输帧格式
+type TransFrame struct {
+	Header     [4]byte // 帧头 E77EE77E
+	SNo        uint32  // 序列号 大端序
+	Reserved0  [2]byte // 保留字节
+	KeyIndex   uint16  // 密钥库索引号 大端序
+	SecretFlag byte    // 明密标志 0x55 明传 0xAA 密传
+	FrameFlag  byte    // 帧类型 0x00:智能载荷电箱有效帧 0x55: 智能载荷电箱空帧
+	FileNo     byte    // 文件编号
+	Reserved1  byte    // 保留字节 0x00
+	Data       [8176]byte
+}
+
+func (t *TransFrame) Decode(frame []byte) {
+	if len(frame) < ImageFrameHeadLength+ImageFrameDataLength {
+		return
+	}
+	copy(t.Header[:], frame[:4])
+	t.SNo = uint32(frame[4])<<24 | uint32(frame[5])<<16 | uint32(frame[6])<<8 | uint32(frame[7])
+	copy(t.Reserved0[:], frame[8:10])
+	t.KeyIndex = uint16(frame[10])<<8 | uint16(frame[11])
+	t.SecretFlag = frame[12]
+	t.FrameFlag = frame[13]
+	t.FileNo = frame[14]
+	t.Reserved1 = frame[15]
+	copy(t.Data[:], frame[16:])
+}