Golang : Process non-XML/JSON formatted ASCII text file example

Problem:

You have to process raw data files produced by third-party servers that you have no control over. The files are not in XML or JSON format... but in plain ASCII text file. How to break the input file down ?

Example of raw input data file.dat:

-

id: 0

name:

en: something

fr: quelquechose

group: group

description:

en: something

fr: quelquechose

hint:

en: something

fr: quelquechose

-

id: 1

name:

en: something1

fr: quelquechose1

group: group1

description:

en: something1

fr: quelquechose1

hint:

en: something1

fr: quelquechose1

Solution:

In this example, we will attempt to break the single large input data file into smaller files based on id number. Basically, what this example solution does is to read line by line and only process line that have a : symbol in it. Please run this example and see the output files for better understanding. This solution will produce simple ASCII text files.

NOTE: You can choose to produce the broken down data into XML or JSON files.

Here you go:

 package main

 import (
 "bufio"
 "fmt"
 "io"
 "os"
 "strings"
 )

 var fname string
 var outputfile *os.File

 func main() {

 file, err := os.Open(`file.dat`)

 if err != nil {
 panic(err)
 }

 defer file.Close()

 reader := bufio.NewReader(file)

 for {
 line, err := reader.ReadString('\n')

 // check if the line has : sign
 // and process the line. Ignore the rest.

 if equal := strings.Index(line, ":"); equal >= 0 {
 if key := strings.TrimSpace(line[:equal]); len(key) > 0 {
 value := ""
 if len(line) > equal {
 value = strings.TrimSpace(line[equal+1:])
 }

 // fmt.Println("Key : ", key, " Value : ", value)

 // filter to different files based on id
 if key == "id" {
 fname = key + value + ".txt"

 //outputfile, err = os.OpenFile(fname, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)

 // without os.O_APPEND will always overwrite the previous id*.txt files if present
 outputfile, err = os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, 0666)

 defer outputfile.Close()

 if err != nil {
 panic(err)
 }

 fmt.Println("Saving to file : ", fname)
 }

 if value != "" {
 io.WriteString(outputfile, key+"="+value+"\n")
 } else {
 io.WriteString(outputfile, "["+key+"]\n")
 }
 }

 }

 if err == io.EOF {
 break
 }

 }

 }

Sample output:

id=0

[name]

en=something

fr=quelquechose

group=group

[description]

en=something

fr=quelquechose

[hint]

en=something

fr=quelquechose

---

id=1

[name]

en=something1

fr=quelquechose1

group=group1

[description]

en=something1

fr=quelquechose1

[hint]

en=something1

fr=quelquechose1

Happy coding!

References:

https://www.socketloop.com/tutorials/golang-text-file-editor-accept-input-from-screen-and-save-to-file

https://www.socketloop.com/tutorials/golang-read-data-from-config-file-and-assign-to-variables