Browse Source

Import prices and CDRs

master
Gerdriaan Mulder 6 years ago
parent
commit
c29855e1c7
  1. 68
      cmd/cdrtool/main.go
  2. 107
      import.go
  3. 139
      pricing.go
  4. 18
      record.go

68
cmd/cdrtool/main.go

@ -0,0 +1,68 @@
package main
import (
"flag"
"fmt"
"log"
"os"
"src.wolkict.net/cdr"
)
var (
csvFile = flag.String("csv", "", "use this call detail record CSV file")
validate = flag.Bool("validate", false, "only validate the call detail record CSV, perform no other actions")
pricesFile = flag.String("prices", "", "use this pricing CSV file")
)
func usage() {
fmt.Println("Usage: cdrtool -csv <file.csv> [-prices <file.csv>] [-validate]")
flag.PrintDefaults()
}
func main() {
flag.Parse()
if *csvFile == "" {
log.Fatalf("mandatory -csv not set")
}
if _, err := os.Open(*csvFile); err != nil {
log.Fatalf("cannot access CSV %q: %v", *csvFile, err)
}
if _, err := os.Open(*pricesFile); err != nil {
log.Printf("[warning] cannot access prices file %q", *pricesFile)
}
imported, err := cdr.ImportCSV(*csvFile)
if *validate {
if err != nil {
log.Fatalf("[validate] importing %q failed: %v", *csvFile, err)
}
log.Printf("[validate] importing %q succeeded", *csvFile)
return
}
if err != nil {
log.Fatalf("importing CSV failed: %v", err)
}
importedPrices, err := cdr.ImportPricesFile(*pricesFile)
if *validate {
if err != nil {
log.Fatalf("[validate] importing %q failed: %v", *pricesFile, err)
}
log.Printf("[validate] importing %q succeeded", *pricesFile)
return
}
if err != nil {
log.Fatalf("importing CSV failed: %v", err)
}
for _, i := range imported {
fmt.Printf("l: %+v\n", i)
}
for _, i := range importedPrices {
fmt.Printf("p: %+v\n", i)
}
}

107
import.go

@ -2,6 +2,11 @@ package cdr
import (
"encoding/csv"
"fmt"
"os"
"strconv"
"strings"
"time"
)
// Some assumptions on our input, since encoding/csv reads everything as string. We expect 17 entries per line:
@ -21,16 +26,18 @@ import (
// - package is free text (or empty)
// - {package,network,service}_costs are either empty or conform to the "costs" column
// FieldsPerLine are the expected number of fields in each CSV line.
const FieldsPerLine = 17
// FieldsPerCDRLine are the expected number of fields in each CSV line.
const FieldsPerCDRLine = 17
// DateTimeFormat is the format of the "time" column, according to time.Time's format
const DateTimeFormat = "2006-01-02 15:04:05"
// ImportIndex keeps track of what column corresponds to what data.
type ImportIndex int
// These are our numbered input fields
const (
ImportUnknown ImportIndex = iota
ImportCid
ImportCid ImportIndex = iota
ImportTime
ImportType
ImportCLI
@ -48,3 +55,95 @@ const (
ImportNetworkCosts
ImportServiceCosts
)
func ImportCSV(fn string) ([]Line, error) {
f, err := os.Open(fn)
if err != nil {
return nil, fmt.Errorf("opening file %q failed: %v", fn, err)
}
csvReader := csv.NewReader(f)
csvReader.FieldsPerRecord = FieldsPerCDRLine
all, err := csvReader.ReadAll()
if err != nil {
return nil, fmt.Errorf("reading csv %q failed: %v", fn, err)
}
ret := make([]Line, len(all))
for i, line := range all {
if line[0] == "cid" {
// Header line
continue
}
// Data type checks
tryDate, err := time.Parse(DateTimeFormat, line[ImportTime])
if err != nil {
return nil, fmt.Errorf("error parsing datetime: %v. offending line: %+v", err, line)
}
tryLineKind, ok := LineKindMap[line[ImportType]]
if !ok {
return nil, fmt.Errorf("error parsing LineKind, unknown kind %q. offending line: %+v", tryLineKind, line)
}
tryReasonKind, ok := ReasonKindMap[line[ImportReason]]
if !ok {
return nil, fmt.Errorf("error parsing ReasonKind, unknown kind %q. offending line: %+v", tryReasonKind, line)
}
tryRawCosts, err := decimalStrToInt(line[ImportCosts], ".")
if err != nil {
return nil, fmt.Errorf("error parsing decimal RawCosts: %v. offending line: %+v", err, line)
}
tryPackageCosts, err := decimalStrToInt(line[ImportPackageCosts], ".")
if err != nil {
return nil, fmt.Errorf("error parsing decimal PackageCosts: %v. offending line: %+v", err, line)
}
tryNetworkCosts, err := decimalStrToInt(line[ImportNetworkCosts], ".")
if err != nil {
return nil, fmt.Errorf("error parsing decimal NetworkCosts: %v. offending line: %+v", err, line)
}
tryServiceCosts, err := decimalStrToInt(line[ImportServiceCosts], ".")
if err != nil {
return nil, fmt.Errorf("error parsing decimal ServiceCosts: %v. offending line: %+v", err, line)
}
tryRawCount, err := strconv.Atoi(line[ImportUsage])
if err != nil {
return nil, fmt.Errorf("error parsing RawCount: %v. offending line: %+v", err, line)
}
tryImportLeg, err := strconv.Atoi(line[ImportLeg])
if err != nil {
return nil, fmt.Errorf("error parsing ImportLeg: %v. offending line: %+v", err, line)
}
ret[i] = Line{
Id: line[ImportCid],
Time: tryDate,
CLI: line[ImportCLI],
From: line[ImportFrom],
To: line[ImportTo],
Account: line[ImportAccount],
Source: line[ImportSource],
Destination: line[ImportDestination],
RawCount: tryRawCount,
RawCosts: tryRawCosts,
Leg: tryImportLeg,
Reason: tryReasonKind,
Kind: tryLineKind,
pkg: line[ImportPackage],
package_cost: tryPackageCosts,
network_cost: tryNetworkCosts,
service_cost: tryServiceCosts,
}
}
return ret, nil
}
// decimalStrToInt removes the decimal separator sep, and converts the remaining string into an integer. If the input is empty, it
// silently converts to 0.
func decimalStrToInt(s, decsep string) (int, error) {
candidate := strings.Replace(s, decsep, "", 1)
if candidate == "" {
return 0, nil
}
return strconv.Atoi(candidate)
}

139
pricing.go

@ -0,0 +1,139 @@
package cdr
import (
"encoding/csv"
"fmt"
"os"
"regexp"
)
type PriceType int
const (
PriceUnknown PriceType = iota
PriceCall
PriceText
PriceData
)
type Price struct {
Type PriceType
BuyEach int
BuyUnit int
SellEach int
SellUnit int
Destination string
}
const FieldsPerPricingLine = 14
// PriceImportIndex keeps track of what column corresponds to what data
type PriceImportIndex int
const (
PriceImportRegion PriceImportIndex = iota
PriceImportDestination
PriceImportPpcBuy
PriceImportPpmBuyFixed
PriceImportPpmBuyMobile
PriceImportPpcSell
PriceImportPpmSellFixed
PriceImportPpmSellMobile
PriceImportPpcMargin
PriceImportPpmMarginFixed
PriceImportPpmMarginMobile
PriceImportPpcSellVAT
PriceImportPpmSellFixedVAT
PriceImportPpmSellMobileVAT
)
var (
PricingRegexp = regexp.MustCompile(`(\d+\.\d+)`)
)
// ImportPricesFile expects a CSV with the following fields:
// - Regio (string)
// - Destination (string)
// - ppc_buy
// - ppm_buy_fixed
// - ppm_buy_mobile
// - ppc_sell
// - ppm_sell_fixed
// - ppm_sell_mobile
// - ppc_margin
// - ppm_margin_fixed
// - ppm_margin_mobile
// - ppc_sell_vat
// - ppm_sell_fixed_vat
// - ppm_sell_mobile_vat
func ImportPricesFile(fn string) ([]Price, error) {
f, err := os.Open(fn)
if err != nil {
return nil, fmt.Errorf("opening file %q failed: %v", fn, err)
}
csvReader := csv.NewReader(f)
csvReader.FieldsPerRecord = FieldsPerPricingLine
csvReader.Comma = ';'
all, err := csvReader.ReadAll()
if err != nil {
return nil, fmt.Errorf("reading csv %q failed: %v", fn, err)
}
var ret []Price
priceTypes := []string{
"Fixed",
"Mobile",
}
for _, line := range all {
if line[PriceImportRegion] == "Regio" || line[PriceImportDestination] == "" {
// Header line or no destination
continue
}
for _, priceType := range priceTypes {
destination := line[PriceImportRegion] + " - " + line[PriceImportDestination] + " - " + priceType
priceKind := PriceCall // TODO: import Text and Data through the same CSV
var buyUnit, sellUnit int
buyEach := convertImportedPrice(line[PriceImportPpcBuy])
sellEach := convertImportedPrice(line[PriceImportPpcSell])
switch priceType {
case "Fixed":
buyUnit = convertImportedPrice(line[PriceImportPpmBuyFixed])
sellUnit = convertImportedPrice(line[PriceImportPpmSellFixed])
case "Mobile":
buyUnit = convertImportedPrice(line[PriceImportPpmBuyMobile])
sellUnit = convertImportedPrice(line[PriceImportPpmSellMobile])
default:
return nil, fmt.Errorf("unsupported priceType %q", priceType)
}
// Extract the prices
ret = append(ret, Price{
Type: priceKind,
Destination: destination,
BuyEach: buyEach,
BuyUnit: buyUnit,
SellEach: sellEach,
SellUnit: sellUnit,
})
}
}
return ret, nil
}
func convertImportedPrice(s string) int {
if s == "" {
return 0
}
p := PricingRegexp.FindString(s)
conv, err := decimalStrToInt(p, ".")
if err != nil {
panic("cannot convert ImportedPrice: " + err.Error())
}
return conv
}

18
record.go

@ -22,6 +22,13 @@ const (
DataLine
)
// LineKindMap translates string from CSV to our typed LineKind
var LineKindMap = map[string]LineKind{
"Voice": VoiceLine,
"Data": DataLine,
"SMS": TextLine,
}
const (
// UnknownReason aka "never seen this reason before"
UnknownReason ReasonKind = iota
@ -41,6 +48,17 @@ const (
PBXOR
)
// ReasonKindMap translates string from CSV to our typed ReasonKind
var ReasonKindMap = map[string]ReasonKind{
"ORIG": ORIG,
"CFIM": CFIM,
"CFNA": CFNA,
"CFBS": CFBS,
"CFOR": CFOR,
"ROAM": ROAM,
"PBXOR": PBXOR,
}
// Line contains the original metadata of a (call) detail record
type Line struct {
Id string

Loading…
Cancel
Save