Import prices and CDRs

cmd/cdrtool/main.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"src.wolkict.net/cdr"
+)
+
+var (
+	csvFile    = flag.String("csv", "", "use this call detail record CSV file")
+	validate   = flag.Bool("validate", false, "only validate the call detail record CSV, perform no other actions")
+	pricesFile = flag.String("prices", "", "use this pricing CSV file")
+)
+
+func usage() {
+	fmt.Println("Usage: cdrtool -csv <file.csv> [-prices <file.csv>] [-validate]")
+	flag.PrintDefaults()
+}
+
+func main() {
+	flag.Parse()
+
+	if *csvFile == "" {
+		log.Fatalf("mandatory -csv not set")
+	}
+
+	if _, err := os.Open(*csvFile); err != nil {
+		log.Fatalf("cannot access CSV %q: %v", *csvFile, err)
+	}
+	if _, err := os.Open(*pricesFile); err != nil {
+		log.Printf("[warning] cannot access prices file %q", *pricesFile)
+	}
+
+	imported, err := cdr.ImportCSV(*csvFile)
+	if *validate {
+		if err != nil {
+			log.Fatalf("[validate] importing %q failed: %v", *csvFile, err)
+		}
+		log.Printf("[validate] importing %q succeeded", *csvFile)
+		return
+	}
+	if err != nil {
+		log.Fatalf("importing CSV failed: %v", err)
+	}
+
+	importedPrices, err := cdr.ImportPricesFile(*pricesFile)
+	if *validate {
+		if err != nil {
+			log.Fatalf("[validate] importing %q failed: %v", *pricesFile, err)
+		}
+		log.Printf("[validate] importing %q succeeded", *pricesFile)
+		return
+	}
+	if err != nil {
+		log.Fatalf("importing CSV failed: %v", err)
+	}
+
+	for _, i := range imported {
+		fmt.Printf("l: %+v\n", i)
+	}
+
+	for _, i := range importedPrices {
+		fmt.Printf("p: %+v\n", i)
+	}
+}

import.go

@@ -2,6 +2,11 @@ package cdr
 
 import (
 	"encoding/csv"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
 )
 
 // Some assumptions on our input, since encoding/csv reads everything as string. We expect 17 entries per line:
@@ -21,16 +26,18 @@ import (
 //   - package is free text (or empty)
 //   - {package,network,service}_costs are either empty or conform to the "costs" column
 
-// FieldsPerLine are the expected number of fields in each CSV line.
+// FieldsPerCDRLine are the expected number of fields in each CSV line.
-const FieldsPerLine = 17
+const FieldsPerCDRLine = 17
+
+// DateTimeFormat is the format of the "time" column, according to time.Time's format
+const DateTimeFormat = "2006-01-02 15:04:05"
 
 // ImportIndex keeps track of what column corresponds to what data.
 type ImportIndex int
 
 // These are our numbered input fields
 const (
-	ImportUnknown ImportIndex = iota
+	ImportCid ImportIndex = iota
-	ImportCid
 	ImportTime
 	ImportType
 	ImportCLI
@@ -48,3 +55,95 @@ const (
 	ImportNetworkCosts
 	ImportServiceCosts
 )
+
+func ImportCSV(fn string) ([]Line, error) {
+	f, err := os.Open(fn)
+	if err != nil {
+		return nil, fmt.Errorf("opening file %q failed: %v", fn, err)
+	}
+
+	csvReader := csv.NewReader(f)
+	csvReader.FieldsPerRecord = FieldsPerCDRLine
+
+	all, err := csvReader.ReadAll()
+	if err != nil {
+		return nil, fmt.Errorf("reading csv %q failed: %v", fn, err)
+	}
+
+	ret := make([]Line, len(all))
+	for i, line := range all {
+		if line[0] == "cid" {
+			// Header line
+			continue
+		}
+		// Data type checks
+		tryDate, err := time.Parse(DateTimeFormat, line[ImportTime])
+		if err != nil {
+			return nil, fmt.Errorf("error parsing datetime: %v. offending line: %+v", err, line)
+		}
+		tryLineKind, ok := LineKindMap[line[ImportType]]
+		if !ok {
+			return nil, fmt.Errorf("error parsing LineKind, unknown kind %q. offending line: %+v", tryLineKind, line)
+		}
+		tryReasonKind, ok := ReasonKindMap[line[ImportReason]]
+		if !ok {
+			return nil, fmt.Errorf("error parsing ReasonKind, unknown kind %q. offending line: %+v", tryReasonKind, line)
+		}
+		tryRawCosts, err := decimalStrToInt(line[ImportCosts], ".")
+		if err != nil {
+			return nil, fmt.Errorf("error parsing decimal RawCosts: %v. offending line: %+v", err, line)
+		}
+		tryPackageCosts, err := decimalStrToInt(line[ImportPackageCosts], ".")
+		if err != nil {
+			return nil, fmt.Errorf("error parsing decimal PackageCosts: %v. offending line: %+v", err, line)
+		}
+		tryNetworkCosts, err := decimalStrToInt(line[ImportNetworkCosts], ".")
+		if err != nil {
+			return nil, fmt.Errorf("error parsing decimal NetworkCosts: %v. offending line: %+v", err, line)
+		}
+		tryServiceCosts, err := decimalStrToInt(line[ImportServiceCosts], ".")
+		if err != nil {
+			return nil, fmt.Errorf("error parsing decimal ServiceCosts: %v. offending line: %+v", err, line)
+		}
+		tryRawCount, err := strconv.Atoi(line[ImportUsage])
+		if err != nil {
+			return nil, fmt.Errorf("error parsing RawCount: %v. offending line: %+v", err, line)
+		}
+		tryImportLeg, err := strconv.Atoi(line[ImportLeg])
+		if err != nil {
+			return nil, fmt.Errorf("error parsing ImportLeg: %v. offending line: %+v", err, line)
+		}
+
+		ret[i] = Line{
+			Id:           line[ImportCid],
+			Time:         tryDate,
+			CLI:          line[ImportCLI],
+			From:         line[ImportFrom],
+			To:           line[ImportTo],
+			Account:      line[ImportAccount],
+			Source:       line[ImportSource],
+			Destination:  line[ImportDestination],
+			RawCount:     tryRawCount,
+			RawCosts:     tryRawCosts,
+			Leg:          tryImportLeg,
+			Reason:       tryReasonKind,
+			Kind:         tryLineKind,
+			pkg:          line[ImportPackage],
+			package_cost: tryPackageCosts,
+			network_cost: tryNetworkCosts,
+			service_cost: tryServiceCosts,
+		}
+	}
+
+	return ret, nil
+}
+
+// decimalStrToInt removes the decimal separator sep, and converts the remaining string into an integer. If the input is empty, it
+// silently converts to 0.
+func decimalStrToInt(s, decsep string) (int, error) {
+	candidate := strings.Replace(s, decsep, "", 1)
+	if candidate == "" {
+		return 0, nil
+	}
+	return strconv.Atoi(candidate)
+}

pricing.go

@@ -0,0 +1,139 @@
+package cdr
+
+import (
+	"encoding/csv"
+	"fmt"
+	"os"
+	"regexp"
+)
+
+type PriceType int
+
+const (
+	PriceUnknown PriceType = iota
+	PriceCall
+	PriceText
+	PriceData
+)
+
+type Price struct {
+	Type        PriceType
+	BuyEach     int
+	BuyUnit     int
+	SellEach    int
+	SellUnit    int
+	Destination string
+}
+
+const FieldsPerPricingLine = 14
+
+// PriceImportIndex keeps track of what column corresponds to what data
+type PriceImportIndex int
+
+const (
+	PriceImportRegion PriceImportIndex = iota
+	PriceImportDestination
+	PriceImportPpcBuy
+	PriceImportPpmBuyFixed
+	PriceImportPpmBuyMobile
+	PriceImportPpcSell
+	PriceImportPpmSellFixed
+	PriceImportPpmSellMobile
+	PriceImportPpcMargin
+	PriceImportPpmMarginFixed
+	PriceImportPpmMarginMobile
+	PriceImportPpcSellVAT
+	PriceImportPpmSellFixedVAT
+	PriceImportPpmSellMobileVAT
+)
+
+var (
+	PricingRegexp = regexp.MustCompile(`(\d+\.\d+)`)
+)
+
+// ImportPricesFile expects a CSV with the following fields:
+//   - Regio (string)
+//   - Destination (string)
+//   - ppc_buy
+//   - ppm_buy_fixed
+//   - ppm_buy_mobile
+//   - ppc_sell
+//   - ppm_sell_fixed
+//   - ppm_sell_mobile
+//   - ppc_margin
+//   - ppm_margin_fixed
+//   - ppm_margin_mobile
+//   - ppc_sell_vat
+//   - ppm_sell_fixed_vat
+//   - ppm_sell_mobile_vat
+
+func ImportPricesFile(fn string) ([]Price, error) {
+	f, err := os.Open(fn)
+	if err != nil {
+		return nil, fmt.Errorf("opening file %q failed: %v", fn, err)
+	}
+
+	csvReader := csv.NewReader(f)
+	csvReader.FieldsPerRecord = FieldsPerPricingLine
+	csvReader.Comma = ';'
+
+	all, err := csvReader.ReadAll()
+	if err != nil {
+		return nil, fmt.Errorf("reading csv %q failed: %v", fn, err)
+	}
+
+	var ret []Price
+	priceTypes := []string{
+		"Fixed",
+		"Mobile",
+	}
+	for _, line := range all {
+		if line[PriceImportRegion] == "Regio"  || line[PriceImportDestination] == "" {
+			// Header line or no destination
+			continue
+		}
+
+		for _, priceType := range priceTypes {
+			destination := line[PriceImportRegion] + " - " + line[PriceImportDestination] + " - " + priceType
+			priceKind := PriceCall // TODO: import Text and Data through the same CSV
+
+			var buyUnit, sellUnit int
+			buyEach := convertImportedPrice(line[PriceImportPpcBuy])
+			sellEach := convertImportedPrice(line[PriceImportPpcSell])
+			switch priceType {
+			case "Fixed":
+				buyUnit = convertImportedPrice(line[PriceImportPpmBuyFixed])
+				sellUnit = convertImportedPrice(line[PriceImportPpmSellFixed])
+			case "Mobile":
+				buyUnit = convertImportedPrice(line[PriceImportPpmBuyMobile])
+				sellUnit = convertImportedPrice(line[PriceImportPpmSellMobile])
+			default:
+				return nil, fmt.Errorf("unsupported priceType %q", priceType)
+			}
+
+			// Extract the prices
+			ret = append(ret, Price{
+				Type:        priceKind,
+				Destination: destination,
+				BuyEach:     buyEach,
+				BuyUnit:     buyUnit,
+				SellEach:    sellEach,
+				SellUnit:    sellUnit,
+			})
+		}
+	}
+	return ret, nil
+}
+
+func convertImportedPrice(s string) int {
+	if s == "" {
+		return 0
+	}
+
+	p := PricingRegexp.FindString(s)
+	conv, err := decimalStrToInt(p, ".")
+	if err != nil {
+		panic("cannot convert ImportedPrice: " + err.Error())
+	}
+	return conv
+}

record.go

@@ -22,6 +22,13 @@ const (
 	DataLine
 )
 
+// LineKindMap translates string from CSV to our typed LineKind
+var LineKindMap = map[string]LineKind{
+	"Voice": VoiceLine,
+	"Data":  DataLine,
+	"SMS":   TextLine,
+}
+
 const (
 	// UnknownReason aka "never seen this reason before"
 	UnknownReason ReasonKind = iota
@@ -41,6 +48,17 @@ const (
 	PBXOR
 )
 
+// ReasonKindMap translates string from CSV to our typed ReasonKind
+var ReasonKindMap = map[string]ReasonKind{
+	"ORIG":  ORIG,
+	"CFIM":  CFIM,
+	"CFNA":  CFNA,
+	"CFBS":  CFBS,
+	"CFOR":  CFOR,
+	"ROAM":  ROAM,
+	"PBXOR": PBXOR,
+}
+
 // Line contains the original metadata of a (call) detail record
 type Line struct {
 	Id          string