package cdr import ( "encoding/csv" "fmt" "os" "strconv" "strings" "time" ) // Some assumptions on our input, since encoding/csv reads everything as string. We expect 17 entries per line: // cid,time,type,cli,from,to,account,usage,costs,source,destination,leg,reason,package,package_costs,network_costs,service_costs // // - cid is free text, could be argued to be unique, but there are no guarantees. // - time is a timestamp formatted YYYY-MM-DD hh:mm:sd // - type is in {Voice,Data,SMS}, of course the header line is "type" // - cli,from should be \d+ (eg 31676012321) // - to is either [0-9*]+ for a (partially shielded/) phonenumber, or .+ for APN or Voicemail, or the likes // - account is free text (non-empty) // - usage is an integer >= 0 // - costs is a float with four decimal digits. We strip the dot and interpret it as an integer, to avoid weird rounding errors during processing // - source,destination are free text, but typically either empty or take the form "word(s) - word(s) - word(s)" // - leg is an integer >= 1 // - reason is in {ORIG,CFIM,CFOR,CFNA,CFBS,ROAM,PBXOR}, but there might be other ones // - package is free text (or empty) // - {package,network,service}_costs are either empty or conform to the "costs" column // FieldsPerCDRLine are the expected number of fields in each CSV line. const FieldsPerCDRLine = 17 // DateTimeFormat is the format of the "time" column, according to time.Time's format const DateTimeFormat = "2006-01-02 15:04:05" // ImportIndex keeps track of what column corresponds to what data. type ImportIndex int // These are our numbered input fields const ( ImportCid ImportIndex = iota ImportTime ImportType ImportCLI ImportFrom ImportTo ImportAccount ImportUsage ImportCosts ImportSource ImportDestination ImportLeg ImportReason ImportPackage ImportPackageCosts ImportNetworkCosts ImportServiceCosts ) func ImportCSV(fn string) ([]Line, error) { f, err := os.Open(fn) if err != nil { return nil, fmt.Errorf("opening file %q failed: %v", fn, err) } defer f.Close() csvReader := csv.NewReader(f) csvReader.FieldsPerRecord = FieldsPerCDRLine all, err := csvReader.ReadAll() if err != nil { return nil, fmt.Errorf("reading csv %q failed: %v", fn, err) } ret := make([]Line, len(all)) for i, line := range all { if line[0] == "cid" { // Header line continue } // Data type checks tryDate, err := time.Parse(DateTimeFormat, line[ImportTime]) if err != nil { return nil, fmt.Errorf("error parsing datetime: %v. offending line: %+v", err, line) } tryLineKind, ok := LineKindMap[line[ImportType]] if !ok { return nil, fmt.Errorf("error parsing LineKind, unknown kind %q. offending line: %+v", tryLineKind, line) } tryReasonKind, ok := ReasonKindMap[line[ImportReason]] if !ok { return nil, fmt.Errorf("error parsing ReasonKind, unknown kind %q. offending line: %+v", tryReasonKind, line) } tryRawCosts, err := decimalStrToInt(line[ImportCosts], ".") if err != nil { return nil, fmt.Errorf("error parsing decimal RawCosts: %v. offending line: %+v", err, line) } tryPackageCosts, err := decimalStrToInt(line[ImportPackageCosts], ".") if err != nil { return nil, fmt.Errorf("error parsing decimal PackageCosts: %v. offending line: %+v", err, line) } tryNetworkCosts, err := decimalStrToInt(line[ImportNetworkCosts], ".") if err != nil { return nil, fmt.Errorf("error parsing decimal NetworkCosts: %v. offending line: %+v", err, line) } tryServiceCosts, err := decimalStrToInt(line[ImportServiceCosts], ".") if err != nil { return nil, fmt.Errorf("error parsing decimal ServiceCosts: %v. offending line: %+v", err, line) } tryRawCount, err := strconv.Atoi(line[ImportUsage]) if err != nil { return nil, fmt.Errorf("error parsing RawCount: %v. offending line: %+v", err, line) } tryImportLeg, err := strconv.Atoi(line[ImportLeg]) if err != nil { return nil, fmt.Errorf("error parsing ImportLeg: %v. offending line: %+v", err, line) } ret[i] = Line{ Id: line[ImportCid], Time: tryDate, CLI: line[ImportCLI], From: line[ImportFrom], To: line[ImportTo], Account: line[ImportAccount], Source: line[ImportSource], Destination: line[ImportDestination], RawCount: tryRawCount, RawCosts: tryRawCosts, Leg: tryImportLeg, Reason: tryReasonKind, Kind: tryLineKind, pkg: line[ImportPackage], package_cost: tryPackageCosts, network_cost: tryNetworkCosts, service_cost: tryServiceCosts, } } return ret, nil } // decimalStrToInt removes the decimal separator sep, and converts the remaining string into an integer. If the input is empty, it // silently converts to 0. func decimalStrToInt(s, decsep string) (int, error) { candidate := strings.Replace(s, decsep, "", 1) if candidate == "" { return 0, nil } return strconv.Atoi(candidate) }