From c36cb9f5109809d2aeb10453f4e387efc0fed776 Mon Sep 17 00:00:00 2001 From: silver0511 Date: Tue, 8 Mar 2022 17:27:51 +0800 Subject: [PATCH] add transaction search logic and update version to 1.5.35 --- controller/v1/search.go | 62 +++ data/service/search.go | 581 +++++++++++++++++++++++++++-- data/service/userSettings.go | 22 ++ migration/jsonschema/jsonschema.go | 72 +--- model/search.go | 46 ++- router/v1/search.go | 1 + utils/heputils/heputils.go | 10 + utils/logger/logger.go | 10 +- version.go | 2 +- 9 files changed, 694 insertions(+), 112 deletions(-) diff --git a/controller/v1/search.go b/controller/v1/search.go index 4181c269..71e83ec6 100644 --- a/controller/v1/search.go +++ b/controller/v1/search.go @@ -272,6 +272,68 @@ func (sc *SearchController) GetTransaction(c echo.Context) error { } +// swagger:operation POST /call/transaction search searchGetTransaction +// +// Returns transaction data based upon filtered json +// --- +// consumes: +// - application/json +// produces: +// - application/json +// parameters: +// - name: SearchObject +// in: body +// type: object +// description: SearchObject parameters +// schema: +// "$ref": "#/definitions/SearchCallData" +// required: true +// SecurityDefinitions: +// bearer: +// type: apiKey +// name: Authorization +// in: header +// responses: +// '200': body:ListUsers +// '400': body:UserLoginFailureResponse +func (sc *SearchController) GetTransactionV2(c echo.Context) error { + transactionObject := model.SearchObject{} + if err := c.Bind(&transactionObject); err != nil { + logger.Error(err.Error()) + return httpresponse.CreateBadResponse(&c, http.StatusBadRequest, webmessages.UserRequestFormatIncorrect) + } + + transactionData, _ := json.Marshal(transactionObject) + correlation, _ := sc.SettingService.GetCorrelationMapsV2(&transactionObject) + aliasRowData, _ := sc.AliasService.GetAllActive() + + aliasData := make(map[string]string) + for _, row := range aliasRowData { + cidr := row.IP + "/" + strconv.Itoa(*row.Mask) + Port := strconv.Itoa(*row.Port) + CaptureID := row.CaptureID + ip, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + logger.Error("ParseCIDR alias CIDR: ["+cidr+"] error: ", err.Error()) + } else { + for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) { + aliasData[ip.String()+":"+Port] = row.Alias + if config.Setting.MAIN_SETTINGS.UseCaptureIDInAlias { + aliasData[ip.String()+":"+Port+":"+CaptureID] = row.Alias + } + } + } + } + + userGroup := auth.GetUserGroup(c) + + reply, _ := sc.SearchService.GetTransactionV2(transactionData, + correlation, aliasData, transactionObject.Param.Location.Node, + sc.SettingService, userGroup, transactionObject.Param.WhiteList) + + return httpresponse.CreateSuccessResponse(&c, http.StatusCreated, reply) +} + // swagger:route POST /call/report/qos search searchGetTransactionQos // // Returns qos data based upon filtered json diff --git a/data/service/search.go b/data/service/search.go index 773e1746..139646ba 100644 --- a/data/service/search.go +++ b/data/service/search.go @@ -2,6 +2,7 @@ package service import ( "bytes" + "container/list" "encoding/json" "errors" "fmt" @@ -135,7 +136,7 @@ func buildQuery(elems []interface{}, orLogic bool, mappingJSON json.RawMessage, //We should be sure that this is value string switch x := formVal.(type) { case string: - formValue = formVal.(string) + formValue = strings.TrimSpace(formVal.(string)) default: logger.Error("Unsupported type:", x, ", Value: ", formVal, "Name:", formName, ", Type: ", formType, "MAPDATA: ", mapData) continue @@ -377,6 +378,14 @@ func (ss *SearchService) SearchData(searchObject *model.SearchObject, aliasData dataArrayValues = append(dataArrayValues, searchFromTime) dataArrayValues = append(dataArrayValues, searchToTime) + /** + * case "INVITE", "ACK", "BYE", "CANCEL", "UPDATE", "PRACK", "REFER", "INFO": + * h.SIP.Profile = "call" + * case "REGISTER": + * h.SIP.Profile = "registration" + * default: + * h.SIP.Profile = "default" + */ for key, _ := range sData.ChildrenMap() { table = "hep_proto_" + key if sData.Exists(key) { @@ -403,6 +412,7 @@ func (ss *SearchService) SearchData(searchObject *model.SearchObject, aliasData ss.Session[session].Debug(). Table(table). Where(sql, dataArrayValues...). + Order("create_date DESC"). Limit(sLimit). Find(&searchTmp) @@ -417,9 +427,11 @@ func (ss *SearchService) SearchData(searchObject *model.SearchObject, aliasData } /* lets sort it */ - sort.Slice(searchData, func(i, j int) bool { - return searchData[i].CreatedDate.Before(searchData[j].CreatedDate) - }) + //sort.Slice(searchData, func(i, j int) bool { + // return searchData[i].CreatedDate.Before(searchData[j].CreatedDate) + //}) + + searchData = uniqueHepTable(searchData) rows, _ := json.Marshal(searchData) data, _ := gabs.ParseJSON(rows) @@ -1126,36 +1138,429 @@ func (ss *SearchService) GetTransaction(table string, data []byte, correlationJS } } +func (ss *SearchService) GetTransactionV2(data []byte, tableMappingSchema []model.TableMappingSchema, + aliasData map[string]string, nodes []string, settingService *UserSettingsService, userGroup string, whitelist []string) (string, error) { + var dataWhere []interface{} + requestData, _ := gabs.ParseJSON(data) + for _, value := range requestData.Search("param", "search").ChildrenMap() { + for _, v := range value.Search("callid").Data().([]interface{}) { + dataWhere = append(dataWhere, v) + } + } + + timeWhereFrom := requestData.S("timestamp", "from").Data().(float64) + timeWhereTo := requestData.S("timestamp", "to").Data().(float64) + timeFrom := time.Unix(int64(timeWhereFrom/float64(time.Microsecond)), 0).UTC() + timeTo := time.Unix(int64(timeWhereTo/float64(time.Microsecond)), 0).UTC() + + dataRow := ss.getCorrelationFromDB(dataWhere, tableMappingSchema, timeFrom, timeTo, nodes, settingService, userGroup, aliasData, whitelist) + + transactionTables := convert2TransactionTables(dataRow) + + //debug log info + logger.Infof("[GetTransactionV2] sorted ttransactionTables = %#v\n", transactionTables) + reply := ss.getTransactionSummaryV2(transactionTables, aliasData) + return reply, nil +} + +func (ss *SearchService) getCorrelationFromDB(dataWhere []interface{}, tableMappingSchema []model.TableMappingSchema, + timeFrom, timeTo time.Time, nodes []string, settingService *UserSettingsService, userGroup string, + aliasData map[string]string, whitelist []string) []model.HepTable { + var dataRow []model.HepTable + duplicateIdFilter := make(map[int64]bool) + duplicateRawFilter := make(map[string]string) + tableArray := [3]string{"hep_proto_1_call", "hep_proto_1_default", "hep_proto_1_registration"} + for index := 0; index < len(tableArray); index++ { + table := tableArray[index] + tempDataRow, _ := ss.GetTransactionData(table, "sid", dataWhere, timeFrom, timeTo, nodes, userGroup, false, whitelist) + if len(tempDataRow) <= 0 { + continue + } + sort.Slice(tempDataRow, func(i, j int) bool { + var protocolHeaderI model.ProtocolHeader + json.Unmarshal(tempDataRow[i].ProtocolHeader, &protocolHeaderI) + var protocolHeaderJ model.ProtocolHeader + json.Unmarshal(tempDataRow[j].ProtocolHeader, &protocolHeaderJ) + + return protocolHeaderI.CaptureID < protocolHeaderJ.CaptureID && tempDataRow[i].CreatedDate.Before(tempDataRow[j].CreatedDate) + }) + + for _, entry := range tempDataRow { + if duplicateIdFilter[entry.Id] { + continue + } + duplicateIdFilter[entry.Id] = true + + var protocolHeader model.ProtocolHeader + json.Unmarshal(entry.ProtocolHeader, &protocolHeader) + captureId, exists := duplicateRawFilter[entry.Raw] + captureStrings := strings.Split(protocolHeader.CaptureID, "_") + //eg: ALIAS-NAME_10.2.38.203 + if len(captureStrings) == 2 { + if _, ok := aliasData[captureStrings[1]]; !ok { + aliasData[captureStrings[1]] = captureStrings[0] + } + } + + if exists && captureId != protocolHeader.CaptureID { + continue + } + duplicateRawFilter[entry.Raw] = protocolHeader.CaptureID + dataRow = append(dataRow, entry) + } + } + + if len(dataRow) <= 0 { + return dataRow + } + + marshalData, _ := json.Marshal(dataRow) + jsonParsed, _ := gabs.ParseJSON(marshalData) + + for index := 0; index < len(tableMappingSchema); index++ { + correlationJSON := tableMappingSchema[index].CorrelationMapping + correlation, _ := gabs.ParseJSON(correlationJSON) + var dataSrcField = make(map[string][]interface{}) + + if len(correlationJSON) > 0 { + // S is shorthand for Search + for _, child := range jsonParsed.Search().Children() { + for _, corrChild := range correlation.Search().Children() { + sf := corrChild.Search("source_field").Data().(string) + nKey := make(map[string][]interface{}) + if strings.Index(sf, ".") > -1 { + elemArray := strings.Split(sf, ".") + switch child.Search(elemArray[0], elemArray[1]).Data().(type) { + case string: + nKey[sf] = append(nKey[sf], child.Search(elemArray[0], elemArray[1]).Data().(string)) + case float64: + nKey[sf] = append(nKey[sf], child.Search(elemArray[0], elemArray[1]).Data().(float64)) + } + } else { + nKey[sf] = append(nKey[sf], child.Search(sf).Data().(string)) + } + if len(nKey) != 0 { + for _, v := range nKey[sf] { + if !function.KeyExits(v, dataSrcField[sf]) { + dataSrcField[sf] = append(dataSrcField[sf], v) + } + } + } + } + } + } + + var foundCidData []string + + for _, corrs := range correlation.Children() { + var from time.Time + var to time.Time + + sourceField := corrs.Search("source_field").Data().(string) + lookupID := corrs.Search("lookup_id").Data().(float64) + lookupProfile := corrs.Search("lookup_profile").Data().(string) + lookupField := corrs.Search("lookup_field").Data().(string) + lookupRange := corrs.Search("lookup_range").Data().([]interface{}) + newWhereData := dataSrcField[sourceField] + likeSearch := false + + if len(newWhereData) == 0 { + continue + } + + table := "hep_proto_" + strconv.FormatFloat(lookupID, 'f', 0, 64) + "_" + lookupProfile + + logger.Printf("[getCorrelationFromDB] search table name = %#v \n", table) + if len(lookupRange) > 0 { + from = timeFrom.Add(time.Duration(lookupRange[0].(float64)) * time.Second).UTC() + to = timeTo.Add(time.Duration(lookupRange[1].(float64)) * time.Second).UTC() + } + if lookupID == 0 { + logger.Error("We need to implement remote call here") + } else { + if sourceField == "data_header.callid" { + + logger.Debug(lookupProfile) + logger.Debug(lookupField) + } + + if corrs.Exists("input_function_js") { + inputFunction := corrs.Search("input_function_js").Data().(string) + logger.Debug("Input function: ", inputFunction) + newDataArray := executeJSInputFunction(inputFunction, newWhereData) + logger.Debug("sid array before JS:", newWhereData) + if newDataArray != nil { + newWhereData = append(newWhereData, newDataArray...) + } + } + + if corrs.Exists("input_script") { + inputScript := corrs.Search("input_script").Data().(string) + logger.Debug("Input function: ", inputScript) + dataScript, err := settingService.GetScriptByParam("scripts", inputScript) + if err == nil { + scriptNew, _ := strconv.Unquote(dataScript) + logger.Debug("OUR script:", scriptNew) + newDataArray := executeJSInputFunction(scriptNew, newWhereData) + logger.Debug("sid array before JS:", newWhereData) + if newDataArray != nil { + newWhereData = append(newWhereData, newDataArray...) + logger.Debug("sid array after JS:", newWhereData) + } + } + } + + if len(foundCidData) > 0 { + for _, v := range foundCidData { + newWhereData = append(newWhereData, v) + } + } + if corrs.Exists("like_search") && corrs.Search("like_search").Data().(bool) { + likeSearch = true + } + + newDataRow, _ := ss.GetTransactionData(table, lookupField, newWhereData, from, to, nodes, userGroup, likeSearch, whitelist) + logger.Printf("[getCorrelationFromDB] search newDataRow count = %d \n", len(newDataRow)) + if corrs.Exists("append_sid") && corrs.Search("append_sid").Data().(bool) { + marshalData, _ = json.Marshal(newDataRow) + jsonParsed, _ = gabs.ParseJSON(marshalData) + for _, value := range jsonParsed.Children() { + elems := value.Search("sid").Data().(string) + if !heputils.ItemExists(foundCidData, elems) { + foundCidData = append(foundCidData, elems) + } + } + } + + for _, entry := range newDataRow { + if duplicateIdFilter[entry.Id] { + continue + } + duplicateIdFilter[entry.Id] = true + + var protocolHeader model.ProtocolHeader + json.Unmarshal(entry.ProtocolHeader, &protocolHeader) + captureId, exists := duplicateRawFilter[entry.Raw] + if exists && captureId != protocolHeader.CaptureID { + continue + } + duplicateRawFilter[entry.Raw] = protocolHeader.CaptureID + dataRow = append(dataRow, entry) + } + + logger.Debug("Correlation data len:", len(dataRow)) + + if corrs.Exists("output_script") { + outputScript := corrs.Search("output_script").Data().(string) + logger.Debug("Output function: ", outputScript) + dataScript, err := settingService.GetScriptByParam("scripts", outputScript) + if err == nil { + scriptNew, _ := strconv.Unquote(dataScript) + logger.Debug("OUR script:", scriptNew) + newDataRaw := executeJSOutputFunction(scriptNew, dataRow) + //logrus.Debug("sid array before JS:", newDataRaw) + if newDataRaw != nil { + dataRow = newDataRaw + //logrus.Debug("sid array after JS:", dataRow) + } + } + } + } + } + } + + return dataRow +} + +func convert2TransactionTables(dataRow []model.HepTable) []*model.TransactionTable { + sort.Slice(dataRow, func(i, j int) bool { + return dataRow[i].CreatedDate.Before(dataRow[j].CreatedDate) + }) + + var transactionTables []*model.TransactionTable + for _, entry := range dataRow { + var protocolHeader model.ProtocolHeader + json.Unmarshal(entry.ProtocolHeader, &protocolHeader) + var dataHeader model.DataHeader + json.Unmarshal(entry.DataHeader, &dataHeader) + + var transactionTable *model.TransactionTable + for _, entryT := range transactionTables { + if strings.Contains(entryT.ViaBranch, dataHeader.ViaBranch) || + strings.Contains(dataHeader.ViaBranch, entryT.ViaBranch) { + transactionTable = entryT + break + } + } + + if transactionTable == nil { + transactionTable = new(model.TransactionTable) + transactionTable.BeginDate = entry.CreatedDate + transactionTable.FromUser = dataHeader.FromUser + transactionTable.ToUser = dataHeader.ToUser + transactionTables = append(transactionTables, transactionTable) + } + + if len(transactionTable.ViaBranch) < len(dataHeader.ViaBranch) { + transactionTable.ViaBranch = dataHeader.ViaBranch + } + + var transactionMethod *model.TransactionMethod + for _, entryM := range transactionTable.TMethods { + if strings.Contains(entryM.Name, dataHeader.Method) { + transactionMethod = entryM + break + } + } + + if transactionMethod == nil { + transactionMethod = new(model.TransactionMethod) + if strings.ToUpper(dataHeader.Method) == "INVITE" || strings.ToUpper(dataHeader.Method) == "100" { + if strings.Contains(entry.Raw, "a=sendonly") || strings.Contains(entry.Raw, "a=recvonly") { + transactionMethod.Name = "INVITE(HOLD)|100" + } else { + transactionMethod.Name = "INVITE|100" + } + + } else { + transactionMethod.Name = dataHeader.Method + } + + transactionMethod.CSeq = dataHeader.CSeq + transactionMethod.BeginDate = entry.CreatedDate + transactionMethod.BodyList = list.New() + transactionTable.TMethods = append(transactionTable.TMethods, transactionMethod) + } + transactionMethod.BodyList.PushBack(entry) + } + + for _, entryT := range transactionTables { + for _, entryM := range entryT.TMethods { + if !strings.Contains(entryT.Name, entryM.Name) { + if len(entryT.Name) <= 0 { + entryT.Name = entryM.Name + } else { + entryT.Name = entryT.Name + "|" + entryM.Name + } + + if strings.HasPrefix(entryM.Name, "4") { + if len(entryT.ErrorName) <= 0 { + entryT.ErrorName = entryM.Name + } else { + entryT.ErrorName = entryT.ErrorName + "|" + entryM.Name + } + } + } + + newBodyList := list.New() + for rightElem := entryM.BodyList.Front(); rightElem != nil; rightElem = rightElem.Next() { + rightHepTable := rightElem.Value.(model.HepTable) + var rightDataHeader model.DataHeader + json.Unmarshal(rightHepTable.DataHeader, &rightDataHeader) + + leftElem := newBodyList.Back() + for ; leftElem != nil; leftElem = leftElem.Prev() { + leftHepTable := leftElem.Value.(model.HepTable) + var leftDataHeader model.DataHeader + json.Unmarshal(leftHepTable.DataHeader, &leftDataHeader) + if isInsertAfter(entryT.ViaBranch, &leftDataHeader, &rightDataHeader) { + break + } + } + + if leftElem == nil { + newBodyList.PushFront(rightHepTable) + } else { + newBodyList.InsertAfter(rightHepTable, leftElem) + } + } + entryM.BodyList = newBodyList + } + } + + return transactionTables +} + +func isInsertAfter(viaBranch string, left *model.DataHeader, right *model.DataHeader) bool { + viaBranchList := strings.Split(viaBranch, ";") + leftViaBranchList := strings.Split(left.ViaBranch, ";") + leftViaCount := len(leftViaBranchList) + if leftViaCount == 1 { + leftViaCount = len(viaBranchList) + for i := 0; i < len(viaBranchList); i++ { + if viaBranchList[i] == left.ViaBranch { + break + } + leftViaCount-- + } + } + rightViaBranchList := strings.Split(right.ViaBranch, ";") + rightViaCount := len(rightViaBranchList) + if rightViaCount == 1 { + rightViaCount = len(viaBranchList) + for i := 0; i < len(viaBranchList); i++ { + if viaBranchList[i] == right.ViaBranch { + break + } + rightViaCount-- + } + } + + if left.Method == right.Method { + if strings.HasSuffix(right.CSeq, right.Method) || left.Method == "100" { + return leftViaCount <= rightViaCount + } + return leftViaCount >= rightViaCount + } + + if strings.ToUpper(left.Method) == "INVITE" && right.Method == "100" { + return leftViaCount <= rightViaCount + } + + if left.Method == "100" && strings.ToUpper(right.Method) == "INVITE" { + return leftViaCount < rightViaCount + } + + //hop by hop + if left.CSeq == right.CSeq && leftViaCount == rightViaCount { + if strings.HasSuffix(left.CSeq, left.Method) { + return true + } + + if strings.HasSuffix(right.CSeq, right.Method) { + return false + } + } + + return true +} + func uniqueHepTable(hepSlice []model.HepTable) []model.HepTable { keys := make(map[string]bool) - list := []model.HepTable{} + keys2 := make(map[string]string) + var list []model.HepTable for _, entry := range hepSlice { - dataKey := strconv.Itoa(entry.Id) + ":" + entry.CreatedDate.String() - if _, value := keys[dataKey]; !value { - keys[dataKey] = true - list = append(list, entry) + dataKey := strconv.FormatInt(entry.Id, 10) + ":" + entry.CreatedDate.String() + if keys[dataKey] { + continue } - } - - if config.Setting.TRANSACTION_SETTINGS.GlobalDeduplicate { + keys[dataKey] = true - logger.Debug("Transaction size after first filter:", len(list)) - keys2 := make(map[string]string) - list2 := []model.HepTable{} - for _, entry := range list { - var protocolHeader map[string]interface{} + if config.Setting.TRANSACTION_SETTINGS.GlobalDeduplicate { + var protocolHeader model.ProtocolHeader json.Unmarshal(entry.ProtocolHeader, &protocolHeader) - dataKey := entry.Raw - if value, exists := keys2[dataKey]; value == protocolHeader["captureId"].(string) || !exists { - keys2[dataKey] = protocolHeader["captureId"].(string) - list2 = append(list2, entry) + captureId, exists := keys2[entry.Raw] + if exists && captureId != protocolHeader.CaptureID { + continue } + + keys2[entry.Raw] = protocolHeader.CaptureID } - logger.Debug("Transaction size after second filter:", len(list2)) - return list2 - } else { - return list + + list = append(list, entry) } + + return list } // this method create new user in the database @@ -1280,7 +1685,7 @@ func (ss *SearchService) getTransactionSummary(data *gabs.Container, aliasData m } if dataElement.Exists("id") { - callElement.ID = dataElement.S("id").Data().(float64) + callElement.ID = int64(dataElement.S("id").Data().(float64)) } if dataElement.Exists("srcIp") { callElement.SrcIP = dataElement.S("srcIp").Data().(string) @@ -1292,10 +1697,10 @@ func (ss *SearchService) getTransactionSummary(data *gabs.Container, aliasData m callElement.DstHost = dataElement.S("dstIp").Data().(string) } if dataElement.Exists("srcPort") { - callElement.SrcPort = heputils.CheckFloatValue(dataElement.S("srcPort").Data()) + callElement.SrcPort = int(heputils.CheckFloatValue(dataElement.S("srcPort").Data())) } if dataElement.Exists("dstPort") { - callElement.DstPort = heputils.CheckFloatValue(dataElement.S("dstPort").Data()) + callElement.DstPort = int(heputils.CheckFloatValue(dataElement.S("dstPort").Data())) } if dataElement.Exists("method") { @@ -1328,7 +1733,7 @@ func (ss *SearchService) getTransactionSummary(data *gabs.Container, aliasData m } if dataElement.Exists("protocol") { - callElement.Protocol = heputils.CheckFloatValue(dataElement.S("protocol").Data()) + callElement.Protocol = int(heputils.CheckFloatValue(dataElement.S("protocol").Data())) } if dataElement.Exists("sid") { callElement.Sid = dataElement.S("sid").Data().(string) @@ -1360,23 +1765,23 @@ func (ss *SearchService) getTransactionSummary(data *gabs.Container, aliasData m } } - callElement.SrcID = callElement.SrcHost + ":" + strconv.FormatFloat(callElement.SrcPort, 'f', 0, 64) - callElement.DstID = callElement.DstHost + ":" + strconv.FormatFloat(callElement.DstPort, 'f', 0, 64) + callElement.SrcID = callElement.SrcHost + ":" + strconv.Itoa(callElement.SrcPort) + callElement.DstID = callElement.DstHost + ":" + strconv.Itoa(callElement.DstPort) - srcIPPort := callElement.SrcIP + ":" + strconv.FormatFloat(callElement.SrcPort, 'f', 0, 64) - dstIPPort := callElement.DstIP + ":" + strconv.FormatFloat(callElement.DstPort, 'f', 0, 64) + srcIPPort := callElement.SrcIP + ":" + strconv.Itoa(callElement.SrcPort) + dstIPPort := callElement.DstIP + ":" + strconv.Itoa(callElement.DstPort) testInput := net.ParseIP(callElement.SrcHost) if testInput.To4() == nil && testInput.To16() != nil { - srcIPPort = "[" + callElement.SrcIP + "]:" + strconv.FormatFloat(callElement.SrcPort, 'f', 0, 64) - callElement.SrcID = "[" + callElement.SrcHost + "]:" + strconv.FormatFloat(callElement.SrcPort, 'f', 0, 64) + srcIPPort = "[" + callElement.SrcIP + "]:" + strconv.Itoa(callElement.SrcPort) + callElement.SrcID = "[" + callElement.SrcHost + "]:" + strconv.Itoa(callElement.SrcPort) } testInput = net.ParseIP(callElement.DstIP) if testInput.To4() == nil && testInput.To16() != nil { - dstIPPort = "[" + callElement.DstIP + "]:" + strconv.FormatFloat(callElement.DstPort, 'f', 0, 64) - callElement.DstID = "[" + callElement.DstHost + "]:" + strconv.FormatFloat(callElement.DstPort, 'f', 0, 64) + dstIPPort = "[" + callElement.DstIP + "]:" + strconv.Itoa(callElement.DstPort) + callElement.DstID = "[" + callElement.DstHost + "]:" + strconv.Itoa(callElement.DstPort) } srcIPPortZero := callElement.SrcIP + ":" + strconv.Itoa(0) @@ -1469,6 +1874,108 @@ func (ss *SearchService) getTransactionSummary(data *gabs.Container, aliasData m return reply.String() } +func (ss *SearchService) getTransactionSummaryV2(transactionTables []*model.TransactionTable, aliasData map[string]string) string { + alias := gabs.New() + dataReply := gabs.Wrap([]interface{}{}) + var transactionElements []model.TransactionElement + var hosts []string + for _, entryT := range transactionTables { + transactionElement := model.TransactionElement{ + ViaBranch: entryT.ViaBranch, + Name: entryT.Name, + ErrorName: entryT.ErrorName, + BeginDate: entryT.BeginDate.UnixNano() / 1000000, + FromUser: entryT.FromUser, + ToUser: entryT.ToUser, + } + for _, entryM := range entryT.TMethods { + for elem := entryM.BodyList.Front(); elem != nil; elem = elem.Next() { + hepTable := elem.Value.(model.HepTable) + var protocolHeader model.ProtocolHeader + json.Unmarshal(hepTable.ProtocolHeader, &protocolHeader) + var dataHeader model.DataHeader + json.Unmarshal(hepTable.DataHeader, &dataHeader) + callElement := model.CallElement{ + MsgColor: "blue", + Destination: 0, + } + callElement.ID = hepTable.Id + callElement.Sid = hepTable.Sid + callElement.SrcHost = protocolHeader.SrcIP + callElement.SrcIP = protocolHeader.SrcIP + callElement.SrcPort = protocolHeader.SrcPort + callElement.SrcID = protocolHeader.SrcIP + ":" + strconv.Itoa(protocolHeader.SrcPort) + callElement.DstIP = protocolHeader.DstIP + callElement.DstHost = protocolHeader.DstIP + callElement.DstPort = protocolHeader.DstPort + callElement.DstID = protocolHeader.DstIP + ":" + strconv.Itoa(protocolHeader.DstPort) + callElement.Method = dataHeader.Method + callElement.MethodText = dataHeader.Method + callElement.CreateDate = hepTable.CreatedDate.UnixNano() / 1000000 + callElement.MicroTs = hepTable.CreatedDate.UnixNano() / 1000000 + callElement.Protocol = protocolHeader.Protocol + callElement.RuriUser = dataHeader.RuriUser + + if !alias.Exists(callElement.SrcID) { + if valueAlias, okAlias := aliasData[callElement.SrcIP]; okAlias { + alias.Set(valueAlias, callElement.SrcID) + } + } + + if !alias.Exists(callElement.DstID) { + if valueAlias, okAlias := aliasData[callElement.DstIP]; okAlias { + alias.Set(valueAlias, callElement.DstID) + } + } + + transactionElement.CallData = append(transactionElement.CallData, callElement) + dataElement := gabs.New() + marshalData, _ := json.Marshal(hepTable) + jsonParsed, _ := gabs.ParseJSON(marshalData) + for k, v := range jsonParsed.ChildrenMap() { + switch k { + case "data_header", "protocol_header": + dataElement.Merge(v) + case "sid", "correlation_id": + sidData := gabs.New() + sidData.Set(v.Data().(interface{}), k) + dataElement.Merge(sidData) + default: + newData := gabs.New() + newData.Set(v.Data().(interface{}), k) + dataElement.Merge(newData) + } + } + dataReply.ArrayAppend(dataElement.Data()) + + transactionElement.Host = heputils.AppendIfNotExists(transactionElement.Host, callElement.SrcID) + transactionElement.Host = heputils.AppendIfNotExists(transactionElement.Host, callElement.DstID) + + } + } + transactionElements = append(transactionElements, transactionElement) + for indexI := 0; indexI < len(transactionElement.Host); indexI++ { + indexJ := 0 + for ; indexJ < len(hosts); indexJ++ { + if hosts[indexJ] == transactionElement.Host[indexI] { + break + } + } + + if indexJ >= len(hosts) { + hosts = append(hosts, transactionElement.Host[indexI]) + } + } + } + + reply := gabs.New() + reply.Set(dataReply.Data(), "data", "messages") + reply.Set(hosts, "data", "hosts") + reply.Set(transactionElements, "data", "transaction_elements") + reply.Set(alias.Data(), "data", "alias") + return reply.String() +} + // this method create new user in the database // it doesn't check internally whether all the validation are applied or not func (ss *SearchService) GetTransactionQos(tables [2]string, data []byte, nodes []string) (string, error) { diff --git a/data/service/userSettings.go b/data/service/userSettings.go index 50887f5c..f2aa52bc 100644 --- a/data/service/userSettings.go +++ b/data/service/userSettings.go @@ -41,6 +41,28 @@ func (ss *UserSettingsService) GetCorrelationMap(data *model.SearchObject) ([]by return mappingSchema.CorrelationMapping, nil } +func (ss *UserSettingsService) GetCorrelationMapsV2(data *model.SearchObject) ([]model.TableMappingSchema, error) { + + mappingSchema := []model.TableMappingSchema{} + Data, _ := json.Marshal(data.Param.Search) + sData, _ := gabs.ParseJSON(Data) + hepid := 1 + //profile := "call" + + for key := range sData.ChildrenMap() { + dataParse := strings.Split(key, "_") + hepid, _ = strconv.Atoi(dataParse[0]) + //profile = dataParse[1] + } + + ss.Session.Debug(). + Table("mapping_schema"). + Where("hepid = ? and profile in ('call', 'default', 'registration')", hepid). + Find(&mappingSchema) + + return mappingSchema, nil +} + // get all mapping from database func (ss *UserSettingsService) GetAllMapping() (map[string]json.RawMessage, error) { diff --git a/migration/jsonschema/jsonschema.go b/migration/jsonschema/jsonschema.go index 2c7a0f98..3e602067 100644 --- a/migration/jsonschema/jsonschema.go +++ b/migration/jsonschema/jsonschema.go @@ -530,7 +530,7 @@ var FieldsMapping1call = json.RawMessage(`[ { "id": "protocol_header.captureId", "name": "Capture ID", - "type": "integer", + "type": "string", "index": "none", "form_type": "input", "position": 17, @@ -799,7 +799,7 @@ var FieldsMapping1default = json.RawMessage(`[ { "id": "protocol_header.captureId", "name": "Capture ID", - "type": "integer", + "type": "string", "index": "none", "form_type": "input", "position": 17, @@ -1018,7 +1018,7 @@ var FieldsMapping34default = json.RawMessage(`[ { "id": "protocol_header.captureId", "name": "Capture ID", - "type": "integer", + "type": "string", "index": "none", "form_type": "input", "position": 12, @@ -1164,7 +1164,7 @@ var FieldsMapping100default = json.RawMessage(`[ { "id": "protocol_header.captureId", "name": "Capture ID", - "type": "integer", + "type": "string", "index": "none", "form_type": "input", "position": 17, @@ -1350,27 +1350,7 @@ var FieldsMapping2000loki = json.RawMessage(`[ } ]`) -var CorrelationMapping1call = json.RawMessage(`[ - { - "source_field": "data_header.callid", - "lookup_id": 100, - "lookup_profile": "default", - "lookup_field": "sid", - "lookup_range": [ - -300, - 200 - ] - }, - { - "source_field": "data_header.callid", - "lookup_id": 5, - "lookup_profile": "default", - "lookup_field": "sid", - "lookup_range": [ - -300, - 200 - ] - }, +var CorrelationMapping1call = json.RawMessage(`[ { "source_field": "protocol_header.correlation_id", "lookup_id": 1, @@ -1395,27 +1375,7 @@ var CorrelationMapping1call = json.RawMessage(`[ ] `) -var CorrelationMapping1registration = json.RawMessage(`[ - { - "source_field": "data_header.callid", - "lookup_id": 100, - "lookup_profile": "default", - "lookup_field": "sid", - "lookup_range": [ - -300, - 200 - ] - }, - { - "source_field": "data_header.callid", - "lookup_id": 5, - "lookup_profile": "default", - "lookup_field": "sid", - "lookup_range": [ - -300, - 200 - ] - }, +var CorrelationMapping1registration = json.RawMessage(`[ { "source_field": "protocol_header.correlation_id", "lookup_id": 1, @@ -1430,26 +1390,6 @@ var CorrelationMapping1registration = json.RawMessage(`[ `) var CorrelationMapping1default = json.RawMessage(`[ - { - "source_field": "data_header.callid", - "lookup_id": 100, - "lookup_profile": "default", - "lookup_field": "sid", - "lookup_range": [ - -300, - 200 - ] - }, - { - "source_field": "data_header.callid", - "lookup_id": 5, - "lookup_profile": "default", - "lookup_field": "sid", - "lookup_range": [ - -300, - 200 - ] - }, { "source_field": "protocol_header.correlation_id", "lookup_id": 1, diff --git a/model/search.go b/model/search.go index 9aa31c68..cf720f95 100644 --- a/model/search.go +++ b/model/search.go @@ -1,6 +1,7 @@ package model import ( + "container/list" "encoding/json" "os" "time" @@ -60,7 +61,7 @@ type SearchObject struct { } type HepTable struct { - Id int `json:"id"` + Id int64 `json:"id"` Sid string `json:"sid"` CreatedDate time.Time `gorm:"column:create_date" json:"create_date"` ProtocolHeader json.RawMessage `gorm:"column:protocol_header" json:"protocol_header"` @@ -71,6 +72,23 @@ type HepTable struct { Profile string `gorm:"column:-" json:"profile"` } +type TransactionTable struct { + ViaBranch string `json:"via_branch"` + Name string `json:"name"` + ErrorName string `json:"error_name"` + BeginDate time.Time `json:"begin_date"` + FromUser string `json:"from_user"` + ToUser string `json:"to_user"` + TMethods []*TransactionMethod `json:"t_methods"` +} + +type TransactionMethod struct { + Name string `json:"name"` + CSeq string `json:"cseq"` + BeginDate time.Time `json:"begin_date"` + BodyList *list.List `json:"body_list"` +} + type Message struct { Id int `json:"id"` Sid string `json:"sid"` @@ -85,16 +103,18 @@ type ProtocolHeader struct { DstPort int `json:"dstPort"` SrcPort int `json:"srcPort"` Protocol int `json:"protocol"` - CaptureID int `json:"captureId"` + CaptureID string `json:"captureId"` CapturePass string `json:"capturePass"` PayloadType int `json:"payloadType"` TimeSeconds int `json:"timeSeconds"` TimeUseconds int `json:"timeUseconds"` + correlationId string `json:"correlation_id"` ProtocolFamily int `json:"protocolFamily"` } type DataHeader struct { - Callid string `json:"callid"` + CallID string `json:"callid"` + CSeq string `json:"cseq"` Method string `json:"method"` ToTag string `json:"to_tag"` ToUser string `json:"to_user"` @@ -105,6 +125,7 @@ type DataHeader struct { RuriUser string `json:"ruri_user"` UserAgent string `json:"user_agent"` RuriDomain string `json:"ruri_domain"` + ViaBranch string `json:"via_branch"` } /* @@ -149,7 +170,7 @@ type SecondData struct { type CallElement struct { // example: 5162 - ID float64 `json:"id"` + ID int64 `json:"id"` // example: wvn6zg@127.0.0.1 Sid string `json:"sid"` // example: 179.12.245.132 @@ -165,13 +186,13 @@ type CallElement struct { // example: 179.12.245.132 DstIP string `json:"dstIp"` // example: 5060 - SrcPort float64 `json:"srcPort"` + SrcPort int `json:"srcPort"` // example: Client AliasSrc string `json:"aliasSrc"` // example: Support AliasDst string `json:"aliasDst"` // example: 5060 - DstPort float64 `json:"dstPort"` + DstPort int `json:"dstPort"` // example: INVITE Method string `json:"method"` // example: INVITE @@ -179,7 +200,7 @@ type CallElement struct { // example: 1633374982350 CreateDate int64 `json:"create_date"` // example: 17 - Protocol float64 `json:"protocol"` + Protocol int `json:"protocol"` // example: blue MsgColor string `json:"msg_color"` Table string `json:"table"` @@ -191,6 +212,17 @@ type CallElement struct { MicroTs int64 `json:"micro_ts"` } +type TransactionElement struct { + ViaBranch string `json:"via_branch"` + Name string `json:"name"` + ErrorName string `json:"error_name"` + BeginDate int64 `json:"begin_date"` + FromUser string `json:"from_user"` + ToUser string `json:"to_user"` + Host []string `json:"host"` + CallData []CallElement `json:"call_data"` +} + // swagger:model SearchTransactionLog type SearchTransactionLog struct { // example: "2001" diff --git a/router/v1/search.go b/router/v1/search.go index b373ba53..bb5a1d9b 100644 --- a/router/v1/search.go +++ b/router/v1/search.go @@ -27,6 +27,7 @@ func RouteSearchApis(acc *echo.Group, dataSession map[string]*gorm.DB, configSes acc.POST("/search/call/decode/message", src.GetDecodeMessageById) acc.POST("/call/transaction", src.GetTransaction) + acc.POST("/call/transaction/v2", src.GetTransactionV2) acc.POST("/call/report/qos", src.GetTransactionQos) acc.POST("/call/report/log", src.GetTransactionLog) acc.POST("/export/call/messages/pcap", src.GetMessagesAsPCap) diff --git a/utils/heputils/heputils.go b/utils/heputils/heputils.go index 5251e38b..016b04af 100644 --- a/utils/heputils/heputils.go +++ b/utils/heputils/heputils.go @@ -292,6 +292,16 @@ func ItemExists(arr []string, elem string) bool { return false } +func AppendIfNotExists(arr []string, elem string) []string { + for index := range arr { + if arr[index] == elem { + return arr + } + } + + return append(arr, elem) +} + /* check if the element exists */ func ElementExists(arr []string, elem string) bool { diff --git a/utils/logger/logger.go b/utils/logger/logger.go index a7d7eb23..3e968b0b 100644 --- a/utils/logger/logger.go +++ b/utils/logger/logger.go @@ -36,7 +36,7 @@ const ( /* gorm logger for logrus */ func (*GormLogger) Print(v ...interface{}) { if v[0] == "sql" { - logrus.WithFields(logrus.Fields{"module": "gorm", "type": "sql"}).Print(v[3]) + logrus.WithFields(logrus.Fields{"module": "gorm", "type": "sql"}).Print(v[1:]) } if v[0] == "log" { logrus.WithFields(logrus.Fields{"module": "gorm", "type": "log"}).Print(v[2]) @@ -155,6 +155,14 @@ func Info(args ...interface{}) { Logger.Info(args...) } +func Infof(format string, args ...interface{}) { + Logger.Infof(format, args) +} + +func Printf(format string, args ...interface{}) { + Logger.Printf(format, args) +} + func Error(args ...interface{}) { Logger.Error(args...) } diff --git a/version.go b/version.go index d94c9294..868fe795 100644 --- a/version.go +++ b/version.go @@ -1,7 +1,7 @@ package main //VERSION -var VERSION_APPLICATION = "1.4.34" +var VERSION_APPLICATION = "1.5.35" //NAME var NAME_APPLICATION = "homer-app"