lankun/sjy01-preprocessing
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
nuknal
2024-05-08 17:28:20 +08:00
commit 3cd797cd4a
9 changed files with 301 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
bin
demo

9
Makefile Normal file
View File

@@ -0,0 +1,9 @@
.DEFAULT_GOAL := out
export DATE := $(shell date +%Y%m%d-%H:%M:%S)
export COMPILE_LDFLAGS='-s -w -X "main.BuildDate=${DATE}" '
out:
env CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o bin/sjy01-preprocessing-darwin-arm64 -ldflags=${COMPILE_LDFLAGS} *.go
env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/sjy01-preprocessing -ldflags=${COMPILE_LDFLAGS} *.go
clean:
rm -rf ./bin/*

6
aos.go Normal file
View File

@@ -0,0 +1,6 @@
package main
type AOSFrame struct {
}

11
go.mod Normal file
View File

@@ -0,0 +1,11 @@
module starwiz.cn/sjy01/preprocessing
go 1.21.0
require (
github.com/duke-git/lancet/v2 v2.3.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/spf13/cobra v1.8.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a // indirect
)

14
go.sum Normal file
View File

@@ -0,0 +1,14 @@
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/duke-git/lancet/v2 v2.3.0 h1:Ztie0qOnC4QgGYYqmpmQxbxkPcm54kqFXj1bwhiV8zg=
github.com/duke-git/lancet/v2 v2.3.0/go.mod h1:zGa2R4xswg6EG9I6WnyubDbFO/+A/RROxIbXcwryTsc=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a h1:4iLhBPcpqFmylhnkbY3W0ONLUYYkDAW9xMFLfxgsvCw=
golang.org/x/exp v0.0.0-20221208152030-732eee02a75a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

36
main.go Normal file
View File

@@ -0,0 +1,36 @@
package main
import (
"fmt"
"github.com/spf13/cobra"
)
var (
paramsXML string
)
var rootCmd = &cobra.Command{
Use: "SJY01Preprocessing",
Short: "Preprocessing tools for SJY01 dataset",
Long: `Preprocessing tools for SJY01 dataset`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("preprocessing tools for SJY01 dataset")
p := Params{
InputData: "demo/050722.dat",
OutputPath: "demo/output",
TempPath: "demo/temp",
DataId: "050722",
}
// preprocess(&p)
exractImageData(&p)
},
}
func init() {
rootCmd.PersistentFlags().StringVarP(&paramsXML, "params", "p", "params.xml", "parameters file path")
}
func main() {
rootCmd.Execute()
}

12
params.go Normal file
View File

@@ -0,0 +1,12 @@
package main
type Params struct {
InputData string
OutputPath string
Satellite string
DataId string
LogFile string
TempPath string
Report string
Result string
}

179
process.go Normal file
View File

@@ -0,0 +1,179 @@
package main
import (
"bufio"
"encoding/binary"
"fmt"
"io"
"log"
"os"
"path/filepath"
"sort"
"github.com/duke-git/lancet/v2/slice"
)
const (
AOSFrameLength = 1024
)
var AOSSyncWord = []byte{0x1A, 0xCF, 0xFC, 0x1D}
var ImageFrameHead = 0xE77EE77E
func preprocess(params *Params) error {
// extract data from aos frame
fi, err := os.Open(params.InputData)
if err != nil {
log.Println("exract data from", params.InputData, "error:", err.Error())
return err
}
defer fi.Close()
name := filepath.Base(params.InputData)
aosDataFile := filepath.Join(params.TempPath, "aos_data_"+name)
foData, err := os.OpenFile(aosDataFile,
os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777)
if err != nil {
fmt.Println("create data err", err.Error())
return err
}
defer foData.Close()
wData := bufio.NewWriter(foData)
var validFrameCnt int
var errFrameCnt int
iBuf := make([]byte, AOSFrameLength)
r := bufio.NewReader(fi)
for {
correct := true
nI, err := r.Read(iBuf)
if err != nil && err != io.EOF {
panic(err)
}
if nI < AOSFrameLength || err == io.EOF {
break
}
iTag := iBuf[0:4]
if iTag[0] == 0x1A && iTag[1] == 0xCF && iTag[2] == 0xFC && iTag[3] == 0x1D {
if err := ldpcCheck(iBuf); err != nil {
errFrameCnt++
correct = false
} else {
validFrameCnt++
}
} else {
correct = false
errFrameCnt++
}
if !correct {
continue
}
data := iBuf[10:894]
wData.Write(data)
}
log.Println("valid frame cnt:", validFrameCnt)
log.Println("err frame cnt:", errFrameCnt)
return nil
}
func ldpcCheck(frame []byte) error {
return nil
}
func exractImageData(params *Params) ([]string, error) {
var sData []string
name := filepath.Base(params.InputData)
aosDataFile := filepath.Join(params.TempPath, "aos_data_"+name)
aosData, err := os.ReadFile(aosDataFile)
if err != nil {
log.Println("read data from", aosDataFile, "error:", err.Error())
return nil, err
}
snRange := map[int][]int{}
datSet := map[int][]byte{}
var i int
for i < len(aosData) {
if i+4 > len(aosData) {
break
}
if aosData[i] == 0xE7 && aosData[i+1] == 0x7E && aosData[i+2] == 0xE7 && aosData[i+3] == 0x7E {
i += 4 // <- 帧头
serialNum := binary.BigEndian.Uint32(aosData[i : i+4])
i += 4 // <- 流水号
i += 2 // <- 保留位
i += 2 // <- 密钥库索引号
i += 1 // <- 明密标志
flag := aosData[i]
i += 1 // <-帧标志
fileno := int(aosData[i])
snRange[fileno] = append(snRange[fileno], int(serialNum))
i += 1 // <- 文件号
i += 1 // <- 保留
if i+ImageFrameDataLength > len(aosData) {
log.Println("frame length error")
break
}
// 只保留非空帧
if flag != 0x55 {
datSet[fileno] = append(datSet[fileno], aosData[i:i+ImageFrameDataLength]...)
}
i += ImageFrameDataLength // <- 图像数据
if i > len(aosData) {
log.Println("frame length error")
break
}
// fmt.Println("serialNum:", serialNum, "fileno:", fileno)
continue
}
i++
}
for k, v := range datSet {
frameCnt := slice.Unique(snRange[k])
if len(frameCnt) < 4096 {
log.Println("fileno", k, "数据长度不足4096帧")
continue
}
s01 := filepath.Join(params.TempPath, fmt.Sprintf("%s_S%d.dat", params.DataId, k))
fo, err := os.OpenFile(s01,
os.O_RDWR|os.O_CREATE|os.O_APPEND, 0777)
if err != nil {
log.Println("create file", s01, "error:", err.Error())
return nil, err
}
w := bufio.NewWriter(fo)
w.Write(v)
w.Flush()
fo.Close()
sData = append(sData, s01)
}
for k, v := range snRange {
vv := slice.Unique(v)
sort.Slice(vv, func(i, j int) bool { return vv[i] < vv[j] })
fmt.Println("fileno:", k, ", sn range:", vv[0], "-", vv[len(vv)-1], "len", len(vv))
}
return sData, nil
}

32
trans_frame.go Normal file
View File

@@ -0,0 +1,32 @@
package main
const ImageFrameHeadLength = 16
const ImageFrameDataLength = 8176
// 智能载荷电箱与数传单机之间图像数据传输帧格式
type TransFrame struct {
Header [4]byte // 帧头 E77EE77E
SNo uint32 // 序列号 大端序
Reserved0 [2]byte // 保留字节
KeyIndex uint16 // 密钥库索引号 大端序
SecretFlag byte // 明密标志 0x55 明传 0xAA 密传
FrameFlag byte // 帧类型 0x00:智能载荷电箱有效帧 0x55: 智能载荷电箱空帧
FileNo byte // 文件编号
Reserved1 byte // 保留字节 0x00
Data [8176]byte
}
func (t *TransFrame) Decode(frame []byte) {
if len(frame) < ImageFrameHeadLength+ImageFrameDataLength {
return
}
copy(t.Header[:], frame[:4])
t.SNo = uint32(frame[4])<<24 | uint32(frame[5])<<16 | uint32(frame[6])<<8 | uint32(frame[7])
copy(t.Reserved0[:], frame[8:10])
t.KeyIndex = uint16(frame[10])<<8 | uint16(frame[11])
t.SecretFlag = frame[12]
t.FrameFlag = frame[13]
t.FileNo = frame[14]
t.Reserved1 = frame[15]
copy(t.Data[:], frame[16:])
}