cdr/import_cdr.go

package cdr

import (
	"encoding/csv"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"
)

// Some assumptions on our input, since encoding/csv reads everything as string. We expect 17 entries per line:
// cid,time,type,cli,from,to,account,usage,costs,source,destination,leg,reason,package,package_costs,network_costs,service_costs
//
//   - cid is free text, could be argued to be unique, but there are no guarantees.
//   - time is a timestamp formatted YYYY-MM-DD hh:mm:sd
//   - type is in {Voice,Data,SMS}, of course the header line is "type"
//   - cli,from should be \d+ (eg 31676012321)
//   - to is either [0-9*]+ for a (partially shielded/) phonenumber, or .+ for APN or Voicemail, or the likes
//   - account is free text (non-empty)
//   - usage is an integer >= 0
//   - costs is a float with four decimal digits. We strip the dot and interpret it as an integer, to avoid weird rounding errors during processing
//   - source,destination are free text, but typically either empty or take the form "word(s) - word(s) - word(s)"
//   - leg is an integer >= 1
//   - reason is in {ORIG,CFIM,CFOR,CFNA,CFBS,ROAM,PBXOR}, but there might be other ones
//   - package is free text (or empty)
//   - {package,network,service}_costs are either empty or conform to the "costs" column

// FieldsPerCDRLine are the expected number of fields in each CSV line.
const FieldsPerCDRLine = 17

// DateTimeFormat is the format of the "time" column, according to time.Time's format
const DateTimeFormat = "2006-01-02 15:04:05"

// ImportIndex keeps track of what column corresponds to what data.
type ImportIndex int

// These are our numbered input fields
const (
	ImportCid ImportIndex = iota
	ImportTime
	ImportType
	ImportCLI
	ImportFrom
	ImportTo
	ImportAccount
	ImportUsage
	ImportCosts
	ImportSource
	ImportDestination
	ImportLeg
	ImportReason
	ImportPackage
	ImportPackageCosts
	ImportNetworkCosts
	ImportServiceCosts
)

func ImportCSV(fn string) ([]Line, error) {
	f, err := os.Open(fn)
	if err != nil {
		return nil, fmt.Errorf("opening file %q failed: %v", fn, err)
	}
	defer f.Close()

	csvReader := csv.NewReader(f)
	csvReader.FieldsPerRecord = FieldsPerCDRLine

	all, err := csvReader.ReadAll()
	if err != nil {
		return nil, fmt.Errorf("reading csv %q failed: %v", fn, err)
	}

	ret := make([]Line, len(all))
	for i, line := range all {
		if line[0] == "cid" {
			// Header line
			continue
		}
		// Data type checks
		tryDate, err := time.Parse(DateTimeFormat, line[ImportTime])
		if err != nil {
			return nil, fmt.Errorf("error parsing datetime: %v. offending line: %+v", err, line)
		}
		tryLineKind, ok := LineKindMap[line[ImportType]]
		if !ok {
			return nil, fmt.Errorf("error parsing LineKind, unknown kind %q. offending line: %+v", tryLineKind, line)
		}
		tryReasonKind, ok := ReasonKindMap[line[ImportReason]]
		if !ok {
			return nil, fmt.Errorf("error parsing ReasonKind, unknown kind %q. offending line: %+v", tryReasonKind, line)
		}
		tryRawCosts, err := decimalStrToInt(line[ImportCosts], ".")
		if err != nil {
			return nil, fmt.Errorf("error parsing decimal RawCosts: %v. offending line: %+v", err, line)
		}
		tryPackageCosts, err := decimalStrToInt(line[ImportPackageCosts], ".")
		if err != nil {
			return nil, fmt.Errorf("error parsing decimal PackageCosts: %v. offending line: %+v", err, line)
		}
		tryNetworkCosts, err := decimalStrToInt(line[ImportNetworkCosts], ".")
		if err != nil {
			return nil, fmt.Errorf("error parsing decimal NetworkCosts: %v. offending line: %+v", err, line)
		}
		tryServiceCosts, err := decimalStrToInt(line[ImportServiceCosts], ".")
		if err != nil {
			return nil, fmt.Errorf("error parsing decimal ServiceCosts: %v. offending line: %+v", err, line)
		}
		tryRawCount, err := strconv.Atoi(line[ImportUsage])
		if err != nil {
			return nil, fmt.Errorf("error parsing RawCount: %v. offending line: %+v", err, line)
		}
		tryImportLeg, err := strconv.Atoi(line[ImportLeg])
		if err != nil {
			return nil, fmt.Errorf("error parsing ImportLeg: %v. offending line: %+v", err, line)
		}

		ret[i] = Line{
			Id:           line[ImportCid],
			Time:         tryDate, // This field is recorded in UTC
			CLI:          line[ImportCLI],
			From:         line[ImportFrom],
			To:           line[ImportTo],
			Account:      line[ImportAccount],
			Source:       line[ImportSource],
			Destination:  line[ImportDestination],
			RawCount:     tryRawCount,
			RawCosts:     tryRawCosts,
			Leg:          tryImportLeg,
			Reason:       tryReasonKind,
			Kind:         tryLineKind,
			pkg:          line[ImportPackage],
			package_cost: tryPackageCosts,
			network_cost: tryNetworkCosts,
			service_cost: tryServiceCosts,
		}
	}

	return ret, nil
}

// decimalStrToInt removes the decimal separator sep, and converts the remaining string into an integer. If the input is empty, it
// silently converts to 0.
func decimalStrToInt(s, decsep string) (int, error) {
	candidate := strings.Replace(s, decsep, "", 1)
	if candidate == "" {
		return 0, nil
	}
	return strconv.Atoi(candidate)
}