diff --git a/cmd/cdrtool/main.go b/cmd/cdrtool/main.go new file mode 100644 index 0000000..5528bfd --- /dev/null +++ b/cmd/cdrtool/main.go @@ -0,0 +1,68 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + + "src.wolkict.net/cdr" +) + +var ( + csvFile = flag.String("csv", "", "use this call detail record CSV file") + validate = flag.Bool("validate", false, "only validate the call detail record CSV, perform no other actions") + pricesFile = flag.String("prices", "", "use this pricing CSV file") +) + +func usage() { + fmt.Println("Usage: cdrtool -csv [-prices ] [-validate]") + flag.PrintDefaults() +} + +func main() { + flag.Parse() + + if *csvFile == "" { + log.Fatalf("mandatory -csv not set") + } + + if _, err := os.Open(*csvFile); err != nil { + log.Fatalf("cannot access CSV %q: %v", *csvFile, err) + } + if _, err := os.Open(*pricesFile); err != nil { + log.Printf("[warning] cannot access prices file %q", *pricesFile) + } + + imported, err := cdr.ImportCSV(*csvFile) + if *validate { + if err != nil { + log.Fatalf("[validate] importing %q failed: %v", *csvFile, err) + } + log.Printf("[validate] importing %q succeeded", *csvFile) + return + } + if err != nil { + log.Fatalf("importing CSV failed: %v", err) + } + + importedPrices, err := cdr.ImportPricesFile(*pricesFile) + if *validate { + if err != nil { + log.Fatalf("[validate] importing %q failed: %v", *pricesFile, err) + } + log.Printf("[validate] importing %q succeeded", *pricesFile) + return + } + if err != nil { + log.Fatalf("importing CSV failed: %v", err) + } + + for _, i := range imported { + fmt.Printf("l: %+v\n", i) + } + + for _, i := range importedPrices { + fmt.Printf("p: %+v\n", i) + } +} diff --git a/import.go b/import.go index 2fd5e9e..88a88cf 100644 --- a/import.go +++ b/import.go @@ -2,6 +2,11 @@ package cdr import ( "encoding/csv" + "fmt" + "os" + "strconv" + "strings" + "time" ) // Some assumptions on our input, since encoding/csv reads everything as string. We expect 17 entries per line: @@ -21,16 +26,18 @@ import ( // - package is free text (or empty) // - {package,network,service}_costs are either empty or conform to the "costs" column -// FieldsPerLine are the expected number of fields in each CSV line. -const FieldsPerLine = 17 +// FieldsPerCDRLine are the expected number of fields in each CSV line. +const FieldsPerCDRLine = 17 + +// DateTimeFormat is the format of the "time" column, according to time.Time's format +const DateTimeFormat = "2006-01-02 15:04:05" // ImportIndex keeps track of what column corresponds to what data. type ImportIndex int // These are our numbered input fields const ( - ImportUnknown ImportIndex = iota - ImportCid + ImportCid ImportIndex = iota ImportTime ImportType ImportCLI @@ -48,3 +55,95 @@ const ( ImportNetworkCosts ImportServiceCosts ) + +func ImportCSV(fn string) ([]Line, error) { + f, err := os.Open(fn) + if err != nil { + return nil, fmt.Errorf("opening file %q failed: %v", fn, err) + } + + csvReader := csv.NewReader(f) + csvReader.FieldsPerRecord = FieldsPerCDRLine + + all, err := csvReader.ReadAll() + if err != nil { + return nil, fmt.Errorf("reading csv %q failed: %v", fn, err) + } + + ret := make([]Line, len(all)) + for i, line := range all { + if line[0] == "cid" { + // Header line + continue + } + // Data type checks + tryDate, err := time.Parse(DateTimeFormat, line[ImportTime]) + if err != nil { + return nil, fmt.Errorf("error parsing datetime: %v. offending line: %+v", err, line) + } + tryLineKind, ok := LineKindMap[line[ImportType]] + if !ok { + return nil, fmt.Errorf("error parsing LineKind, unknown kind %q. offending line: %+v", tryLineKind, line) + } + tryReasonKind, ok := ReasonKindMap[line[ImportReason]] + if !ok { + return nil, fmt.Errorf("error parsing ReasonKind, unknown kind %q. offending line: %+v", tryReasonKind, line) + } + tryRawCosts, err := decimalStrToInt(line[ImportCosts], ".") + if err != nil { + return nil, fmt.Errorf("error parsing decimal RawCosts: %v. offending line: %+v", err, line) + } + tryPackageCosts, err := decimalStrToInt(line[ImportPackageCosts], ".") + if err != nil { + return nil, fmt.Errorf("error parsing decimal PackageCosts: %v. offending line: %+v", err, line) + } + tryNetworkCosts, err := decimalStrToInt(line[ImportNetworkCosts], ".") + if err != nil { + return nil, fmt.Errorf("error parsing decimal NetworkCosts: %v. offending line: %+v", err, line) + } + tryServiceCosts, err := decimalStrToInt(line[ImportServiceCosts], ".") + if err != nil { + return nil, fmt.Errorf("error parsing decimal ServiceCosts: %v. offending line: %+v", err, line) + } + tryRawCount, err := strconv.Atoi(line[ImportUsage]) + if err != nil { + return nil, fmt.Errorf("error parsing RawCount: %v. offending line: %+v", err, line) + } + tryImportLeg, err := strconv.Atoi(line[ImportLeg]) + if err != nil { + return nil, fmt.Errorf("error parsing ImportLeg: %v. offending line: %+v", err, line) + } + + ret[i] = Line{ + Id: line[ImportCid], + Time: tryDate, + CLI: line[ImportCLI], + From: line[ImportFrom], + To: line[ImportTo], + Account: line[ImportAccount], + Source: line[ImportSource], + Destination: line[ImportDestination], + RawCount: tryRawCount, + RawCosts: tryRawCosts, + Leg: tryImportLeg, + Reason: tryReasonKind, + Kind: tryLineKind, + pkg: line[ImportPackage], + package_cost: tryPackageCosts, + network_cost: tryNetworkCosts, + service_cost: tryServiceCosts, + } + } + + return ret, nil +} + +// decimalStrToInt removes the decimal separator sep, and converts the remaining string into an integer. If the input is empty, it +// silently converts to 0. +func decimalStrToInt(s, decsep string) (int, error) { + candidate := strings.Replace(s, decsep, "", 1) + if candidate == "" { + return 0, nil + } + return strconv.Atoi(candidate) +} diff --git a/pricing.go b/pricing.go new file mode 100644 index 0000000..54869cf --- /dev/null +++ b/pricing.go @@ -0,0 +1,139 @@ +package cdr + +import ( + "encoding/csv" + "fmt" + "os" + "regexp" +) + +type PriceType int + +const ( + PriceUnknown PriceType = iota + PriceCall + PriceText + PriceData +) + +type Price struct { + Type PriceType + BuyEach int + BuyUnit int + SellEach int + SellUnit int + Destination string +} + +const FieldsPerPricingLine = 14 + +// PriceImportIndex keeps track of what column corresponds to what data +type PriceImportIndex int + +const ( + PriceImportRegion PriceImportIndex = iota + PriceImportDestination + PriceImportPpcBuy + PriceImportPpmBuyFixed + PriceImportPpmBuyMobile + PriceImportPpcSell + PriceImportPpmSellFixed + PriceImportPpmSellMobile + PriceImportPpcMargin + PriceImportPpmMarginFixed + PriceImportPpmMarginMobile + PriceImportPpcSellVAT + PriceImportPpmSellFixedVAT + PriceImportPpmSellMobileVAT +) + +var ( + PricingRegexp = regexp.MustCompile(`(\d+\.\d+)`) +) + +// ImportPricesFile expects a CSV with the following fields: +// - Regio (string) +// - Destination (string) +// - ppc_buy +// - ppm_buy_fixed +// - ppm_buy_mobile +// - ppc_sell +// - ppm_sell_fixed +// - ppm_sell_mobile +// - ppc_margin +// - ppm_margin_fixed +// - ppm_margin_mobile +// - ppc_sell_vat +// - ppm_sell_fixed_vat +// - ppm_sell_mobile_vat + +func ImportPricesFile(fn string) ([]Price, error) { + f, err := os.Open(fn) + if err != nil { + return nil, fmt.Errorf("opening file %q failed: %v", fn, err) + } + + csvReader := csv.NewReader(f) + csvReader.FieldsPerRecord = FieldsPerPricingLine + csvReader.Comma = ';' + + all, err := csvReader.ReadAll() + if err != nil { + return nil, fmt.Errorf("reading csv %q failed: %v", fn, err) + } + + var ret []Price + priceTypes := []string{ + "Fixed", + "Mobile", + } + for _, line := range all { + if line[PriceImportRegion] == "Regio" || line[PriceImportDestination] == "" { + // Header line or no destination + continue + } + + for _, priceType := range priceTypes { + destination := line[PriceImportRegion] + " - " + line[PriceImportDestination] + " - " + priceType + priceKind := PriceCall // TODO: import Text and Data through the same CSV + + var buyUnit, sellUnit int + buyEach := convertImportedPrice(line[PriceImportPpcBuy]) + sellEach := convertImportedPrice(line[PriceImportPpcSell]) + switch priceType { + case "Fixed": + buyUnit = convertImportedPrice(line[PriceImportPpmBuyFixed]) + sellUnit = convertImportedPrice(line[PriceImportPpmSellFixed]) + case "Mobile": + buyUnit = convertImportedPrice(line[PriceImportPpmBuyMobile]) + sellUnit = convertImportedPrice(line[PriceImportPpmSellMobile]) + default: + return nil, fmt.Errorf("unsupported priceType %q", priceType) + } + + // Extract the prices + ret = append(ret, Price{ + Type: priceKind, + Destination: destination, + BuyEach: buyEach, + BuyUnit: buyUnit, + SellEach: sellEach, + SellUnit: sellUnit, + }) + } + } + return ret, nil +} + +func convertImportedPrice(s string) int { + if s == "" { + return 0 + } + + p := PricingRegexp.FindString(s) + conv, err := decimalStrToInt(p, ".") + if err != nil { + panic("cannot convert ImportedPrice: " + err.Error()) + } + return conv +} diff --git a/record.go b/record.go index 603a245..f68b611 100644 --- a/record.go +++ b/record.go @@ -22,6 +22,13 @@ const ( DataLine ) +// LineKindMap translates string from CSV to our typed LineKind +var LineKindMap = map[string]LineKind{ + "Voice": VoiceLine, + "Data": DataLine, + "SMS": TextLine, +} + const ( // UnknownReason aka "never seen this reason before" UnknownReason ReasonKind = iota @@ -41,6 +48,17 @@ const ( PBXOR ) +// ReasonKindMap translates string from CSV to our typed ReasonKind +var ReasonKindMap = map[string]ReasonKind{ + "ORIG": ORIG, + "CFIM": CFIM, + "CFNA": CFNA, + "CFBS": CFBS, + "CFOR": CFOR, + "ROAM": ROAM, + "PBXOR": PBXOR, +} + // Line contains the original metadata of a (call) detail record type Line struct { Id string