diff --git a/VERSION b/VERSION index 9325c3c..60a2d3e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 \ No newline at end of file +0.4.0 \ No newline at end of file diff --git a/code42/auth.go b/code42/auth.go new file mode 100644 index 0000000..2d6c91e --- /dev/null +++ b/code42/auth.go @@ -0,0 +1,77 @@ +package code42 + +import ( + "encoding/json" + "errors" + "io/ioutil" + "net/http" + "strings" +) + +// Code42 Auth + +//Structs of Crashplan FFS API Authentication Token Return +type AuthData struct { + Data AuthToken `json:"data"` + Error string `json:"error,omitempty"` + Warnings string `json:"warnings,omitempty"` +} +type AuthToken struct { + V3UserToken string `json:"v3_user_token"` +} + +/* +GetAuthData - Function to get the Authentication data (mainly the authentication token) which will be needed for the rest of the API calls +The authentication token is good for up to 1 hour before it expires +*/ +func GetAuthData(uri string, username string, password string) (*AuthData, error) { + //Build HTTP GET request + req, err := http.NewRequest("GET", uri, nil) + + //Return nil and err if Building of HTTP GET request fails + if err != nil { + return nil, err + } + + //Set Basic Auth Header + req.SetBasicAuth(username, password) + //Set Accept Header + req.Header.Set("Accept", "application/json") + + //Make the HTTP Call + resp, err := http.DefaultClient.Do(req) + + //Return nil and err if Building of HTTP GET request fails + if err != nil { + return nil, err + } + + defer resp.Body.Close() + + //Return err if status code != 200 + if resp.StatusCode != http.StatusOK { + return nil, errors.New("Error with Authentication Token GET: " + resp.Status) + } + + //Create AuthData variable + var authData AuthData + + respData := resp.Body + + responseBytes, _ := ioutil.ReadAll(respData) + + if strings.Contains(string(responseBytes), "Service Under Maintenance") { + return nil, errors.New("error: auth api service is under maintenance") + } + + //Decode the resp.Body into authData variable + err = json.Unmarshal(responseBytes, &authData) + + //Return nil and err if decoding of resp.Body fails + if err != nil { + return nil, err + } + + //Return AuthData + return &authData, nil +} \ No newline at end of file diff --git a/ffs.go b/code42/csvExport.go similarity index 81% rename from ffs.go rename to code42/csvExport.go index 91d7e86..37e8f56 100644 --- a/ffs.go +++ b/code42/csvExport.go @@ -1,14 +1,10 @@ -//Packages provide a module for using the Code42 Crashplan FFS API -package ffs +package code42 import ( - "bytes" "encoding/csv" "encoding/hex" - "encoding/json" "errors" "github.com/spkg/bom" - "io/ioutil" "log" "net/http" "strconv" @@ -17,8 +13,10 @@ import ( "time" ) -//The main body of a file event record -type FileEvent struct { +// FFS CSV Export + +//The CSV main body of a file event record +type CsvFileEvent struct { EventId string `json:"eventId,omitempty"` EventType string `json:"eventType,omitempty"` EventTimestamp *time.Time `json:"eventTimestamp,omitempty"` @@ -91,102 +89,14 @@ type FileEvent struct { //Currently recognized csv headers var csvHeaders = []string{"Event ID", "Event type", "Date Observed (UTC)", "Date Inserted (UTC)", "File path", "Filename", "File type", "File Category", "Identified Extension Category", "Current Extension Category", "File size (bytes)", "File Owner", "MD5 Hash", "SHA-256 Hash", "Create Date", "Modified Date", "Username", "Device ID", "User UID", "Hostname", "Fully Qualified Domain Name", "IP address (public)", "IP address (private)", "Actor", "Directory ID", "Source", "URL", "Shared", "Shared With Users", "File exposure changed to", "Cloud drive ID", "Detection Source Alias", "File Id", "Exposure Type", "Process Owner", "Process Name", "Tab/Window Title", "Tab URL", "Table Titles", "Tab URLs", "Removable Media Vendor", "Removable Media Name", "Removable Media Serial Number", "Removable Media Capacity", "Removable Media Bus Type", "Removable Media Media Name", "Removable Media Volume Name", "Removable Media Partition Id", "Sync Destination", "Sync Destination Username", "Email DLP Policy Names", "Email DLP Subject", "Email DLP Sender", "Email DLP From", "Email DLP Recipients", "Outside Active Hours", "Identified Extension MIME Type", "Current Extension MIME Type", "Suspicious File Type Mismatch", "Print Job Name", "Printer Name", "Printed Files Backup Path", "Remote Activity", "Trusted", "Logged in Operating System User", "Destination Category", "Destination Name"} -//Structs of Crashplan FFS API Authentication Token Return -type AuthData struct { - Data AuthToken `json:"data"` - Error string `json:"error,omitempty"` - Warnings string `json:"warnings,omitempty"` -} -type AuthToken struct { - V3UserToken string `json:"v3_user_token"` -} - -//Structs for FFS Queries -type Query struct { - Groups []Group `json:"groups"` - GroupClause string `json:"groupClause,omitempty"` - PgNum int `json:"pgNum,omitempty"` - PgSize int `json:"pgSize,omitempty"` - SrtDir string `json:"srtDir,omitempty"` - SrtKey string `json:"srtKey,omitempty"` -} - -type Group struct { - Filters []Filter `json:"filters"` - FilterClause string `json:"filterClause,omitempty"` -} - -type Filter struct { - Operator string `json:"operator"` - Term string `json:"term"` - Value string `json:"value"` -} - -/* -GetAuthData - Function to get the Authentication data (mainly the authentication token) which will be needed for the rest of the API calls -The authentication token is good for up to 1 hour before it expires -*/ -func GetAuthData(uri string, username string, password string) (*AuthData, error) { - //Build HTTP GET request - req, err := http.NewRequest("GET", uri, nil) - - //Return nil and err if Building of HTTP GET request fails - if err != nil { - return nil, err - } - - //Set Basic Auth Header - req.SetBasicAuth(username, password) - //Set Accept Header - req.Header.Set("Accept", "application/json") - - //Make the HTTP Call - resp, err := http.DefaultClient.Do(req) - - //Return nil and err if Building of HTTP GET request fails - if err != nil { - return nil, err - } - - defer resp.Body.Close() - - //Return err if status code != 200 - if resp.StatusCode != http.StatusOK { - return nil, errors.New("Error with Authentication Token GET: " + resp.Status) - } - - //Create AuthData variable - var authData AuthData - - respData := resp.Body - - responseBytes, _ := ioutil.ReadAll(respData) - - if strings.Contains(string(responseBytes), "Service Under Maintenance") { - return nil, errors.New("error: auth api service is under maintenance") - } - - //Decode the resp.Body into authData variable - err = json.Unmarshal(responseBytes, &authData) - - //Return nil and err if decoding of resp.Body fails - if err != nil { - return nil, err - } - - //Return AuthData - return &authData, nil -} - -//TODO create Global Function for calling getFileEvents with CSV url formatting (Priority, as will likely continue to be supported by Code42) /* -csvLineToFileEvent - Converts a CSV Line into a File Event Struct +csvLineToCsvFileEvent - Converts a CSV Line into a File Event Struct []string - csv line. DO NOT PASS Line 0 (Headers) if they exist This function contains panics in order to prevent messed up CSV parsing */ -func csvLineToFileEvent(csvLine []string) *FileEvent { +func csvLineToCsvFileEvent(csvLine []string) *CsvFileEvent { //Init variables - var fileEvent FileEvent + var fileEvent CsvFileEvent var err error //set eventId @@ -591,57 +501,13 @@ func csvLineToFileEvent(csvLine []string) *FileEvent { return &fileEvent } -//TODO create Global Function for calling getFileEvents with JSON url formatting (this may be not be needed, Code42 seems to frown upon using this for pulling large amounts of events.) - /* -getFileEvents - Function to get the actual event records from FFS -authData - authData struct which contains the authentication API token -ffsURI - the URI for where to pull the FFS events -query - query struct which contains the actual FFS query and a golang valid form +getCsvFileEvents - Function to get the actual event records from FFS +*http.Response from ExecQuery This function contains a panic if the csv columns do not match the currently specified list. This is to prevent data from being messed up during parsing. */ -func GetFileEvents(authData AuthData, ffsURI string, query Query) (*[]FileEvent, error) { - - //Validate jsonQuery is valid JSON - ffsQuery, err := json.Marshal(query) - if err != nil { - return nil, errors.New("jsonQuery is not in a valid json format") - } - - //Make sure authData token is not "" - if authData.Data.V3UserToken == "" { - return nil, errors.New("authData cannot be nil") - } - - //Query ffsURI with authData API token and jsonQuery body - req, err := http.NewRequest("POST", ffsURI, bytes.NewReader(ffsQuery)) - - //Handle request errors - if err != nil { - return nil, err - } - - //Set request headers - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "v3_user_token "+authData.Data.V3UserToken) - - //Get Response - resp, err := http.DefaultClient.Do(req) - - //Handle response errors - if err != nil { - return nil, err - } - - //defer body close - defer resp.Body.Close() - - //Make sure http status code is 200 - if resp.StatusCode != http.StatusOK { - return nil, errors.New("Error with gathering file events POST: " + resp.Status) - } - +func GetCsvFileEvents(resp *http.Response) (*[]CsvFileEvent, error) { //Read Response Body as CSV //reader := csv.NewReader(resp.Body) reader := csv.NewReader(bom.NewReader(resp.Body)) @@ -655,7 +521,7 @@ func GetFileEvents(authData AuthData, ffsURI string, query Query) (*[]FileEvent, return nil, err } - var fileEvents []FileEvent + var fileEvents []CsvFileEvent //Loop through CSV lines var wg sync.WaitGroup @@ -664,7 +530,7 @@ func GetFileEvents(authData AuthData, ffsURI string, query Query) (*[]FileEvent, for lineNumber, lineContent := range data { if lineNumber != 0 { //Convert CSV line to file events and add to slice - fileEvents = append(fileEvents, *csvLineToFileEvent(lineContent)) + fileEvents = append(fileEvents, *csvLineToCsvFileEvent(lineContent)) } else { //Validate that the columns have not changed err = equal(lineContent, csvHeaders) @@ -729,4 +595,4 @@ func equal(slice1 []string, slice2 []string) error { } return nil -} +} \ No newline at end of file diff --git a/code42/ffsQuery.go b/code42/ffsQuery.go new file mode 100644 index 0000000..cceff78 --- /dev/null +++ b/code42/ffsQuery.go @@ -0,0 +1,79 @@ +package code42 + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" +) + +//Structs for FFS Queries +type Query struct { + Groups []Group `json:"groups"` + GroupClause string `json:"groupClause,omitempty"` + PgNum int `json:"pgNum,omitempty"` + PgSize int `json:"pgSize,omitempty"` + PgToken *string `json:"pgToken,omitempty"` + SrtDir string `json:"srtDir,omitempty"` + SrtKey string `json:"srtKey,omitempty"` +} + +type Group struct { + Filters []SearchFilter `json:"filters"` + FilterClause string `json:"filterClause,omitempty"` +} + +type SearchFilter struct { + Operator string `json:"operator"` + Term string `json:"term"` + Value string `json:"value"` +} + +type QueryProblem struct { + BadFilter SearchFilter `json:"badFilter,omitempty"` + Description string `json:"description,omitempty"` + Type string `json:"type,omitempty"` +} + +func ExecQuery(authData AuthData, ffsURI string, query Query) (*http.Response, error) { + //Validate jsonQuery is valid JSON + ffsQuery, err := json.Marshal(query) + if err != nil { + return nil, errors.New("jsonQuery is not in a valid json format") + } + + //Make sure authData token is not "" + if authData.Data.V3UserToken == "" { + return nil, errors.New("authData cannot be nil") + } + + //Query ffsURI with authData API token and jsonQuery body + req, err := http.NewRequest("POST", ffsURI, bytes.NewReader(ffsQuery)) + + //Handle request errors + if err != nil { + return nil, err + } + + //Set request headers + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "v3_user_token "+authData.Data.V3UserToken) + + //Get Response + resp, err := http.DefaultClient.Do(req) + + //Handle response errors + if err != nil { + return nil, err + } + + //defer body close + defer resp.Body.Close() + + //Make sure http status code is 200 + if resp.StatusCode != http.StatusOK { + return nil, errors.New("Error with gathering file events POST: " + resp.Status) + } + + return resp, nil +} diff --git a/code42/jsonExport.go b/code42/jsonExport.go new file mode 100644 index 0000000..5bb2e5e --- /dev/null +++ b/code42/jsonExport.go @@ -0,0 +1,181 @@ +package code42 + +import ( + "encoding/json" + "errors" + "io/ioutil" + "net/http" +) + +type JsonFileEvent struct { + Actor string `json:"actor,omitempty"` + CloudDriveId string `json:"cloudDriveId,omitempty"` + CreateTimestamp string `json:"createTimestamp,omitempty"` + DestinationCategory string `json:"destinationCategory,omitempty"` + DestinationName string `json:"destinationName,omitempty"` + DetectionSourceAlias string `json:"detectionSourceAlias,omitempty"` + DeviceUid string `json:"deviceUid,omitempty"` + DeviceUserName string `json:"deviceUserName,omitempty"` + DirectoryId []string `json:"directoryId,omitempty"` + DomainName string `json:"domainName,omitempty"` + EmailDlpPolicyNames []string `json:"emailDlpPolicyNames,omitempty"` + EmailFrom string `json:"emailFrom,omitempty"` + EmailRecipients []string `json:"emailRecipients,omitempty"` + EmailSender string `json:"emailSender,omitempty"` + EmailSubject string `json:"emailSubject,omitempty"` + EventId string `json:"eventId"` + EventTimestamp string `json:"eventTimestamp,omitempty"` + EventType string `json:"eventType,omitempty"` + Exposure []string `json:"exposure,omitempty"` + FieldErrors []FieldError `json:"fieldErrors,omitempty"` + FileCategory string `json:"fileCategory,omitempty"` + FileCategoryByBytes string `json:"fileCategoryByBytes,omitempty"` + FileCategoryByExtension string `json:"fileCategoryByExtension,omitempty"` + FileId string `json:"fileId,omitempty"` + FileName string `json:"fileName,omitempty"` + FileOwner string `json:"fileOwner,omitempty"` + FilePath string `json:"filePath,omitempty"` + FileSize *int64 `json:"fileSize,omitempty"` + FileType string `json:"fileType,omitempty"` + InsertionTimestamp string `json:"insertionTimestamp,omitempty"` + Md5Checksum string `json:"md5Checksum,omitempty"` + MimeTypeByBytes string `json:"mimeTypeByBytes,omitempty"` + MimeTypeByExtension string `json:"mimeTypeByExtension,omitempty"` + MimeTypeMismatch *bool `json:"mimeTypeMismatch,omitempty"` + ModifyTimestamp string `json:"modifyTimestamp,omitempty"` + OperatingSystemUser string `json:"operatingSystemUser,omitempty"` + OsHostName string `json:"osHostName,omitempty"` + OutsideActiveHours *bool `json:"outsideActiveHours,omitempty"` + PrintJobName string `json:"printJobName,omitempty"` + PrinterName string `json:"printerName,omitempty"` + PrivateIpAddresses []string `json:"privateIpAddresses,omitempty"` + ProcessName string `json:"processName,omitempty"` + ProcessOwner string `json:"processOwner,omitempty"` + PublicIpAddress string `json:"publicIpAddress,omitempty"` + RemoteActivity string `json:"remoteActivity,omitempty"` + RemovableMediaBusType string `json:"removableMediaBusType,omitempty"` + RemovableMediaCapacity *int64 `json:"removableMediaCapacity,omitempty"` + RemovableMediaMediaName string `json:"removableMediaMediaName,omitempty"` + RemovableMediaName string `json:"removableMediaName,omitempty"` + RemovableMediaPartitionId []string `json:"removableMediaPartitionId,omitempty"` + RemovableMediaSerialNumber string `json:"removableMediaSerialNumber,omitempty"` + RemovableMediaVendor string `json:"removableMediaVendor,omitempty"` + RemovableMediaVolumeName []string `json:"removableMediaVolumeName,omitempty"` + Sha256Checksum string `json:"sha256Checksum,omitempty"` + Shared string `json:"shared,omitempty"` + SharedWith *SharedWith `json:"sharedWith,omitempty"` + SharingTypeAdded []string `json:"sharingTypeAdded,omitempty"` + Source string `json:"source,omitempty"` + SyncDestination string `json:"syncDestination,omitempty"` + SyncDestinationUsername string `json:"syncDestinationUsername,omitempty"` + TabUrl string `json:"tabUrl,omitempty"` + Tabs []Tab `json:"tabs,omitempty"` + Trusted *bool `json:"trusted,omitempty"` + Url string `json:"url,omitempty"` + UserUid string `json:"userUid,omitempty"` + WindowTitle []string `json:"windowTitle,omitempty"` +} + +type FieldError struct { + Error string `json:"error,omitempty"` + Field string `json:"field,omitempty"` +} + +type SharedWith struct { + CloudUsername string `json:"cloudUsername,omitempty"` +} + +type Tab struct { + Title string `json:"title,omitempty"` + Url string `json:"url,omitempty"` +} + +type JsonFileEventResponse struct { + FileEvents []JsonFileEvent `json:"fileEvents,omitempty"` + NextPgToken *string `json:"nextPgToken,omitempty"` + Problems []QueryProblem `json:"problems,omitempty"` + TotalCount *int64 `json:"totalCount,omitempty"` +} + +func GetJsonFileEventResponse(resp *http.Response) (*JsonFileEventResponse, error) { + var eventResponse JsonFileEventResponse + + //Read Response Body as JSON + body, err := ioutil.ReadAll(resp.Body) + + if err != nil { + return nil, err + } + + err = json.Unmarshal(body, &eventResponse) + + if err != nil { + return nil, err + } + + return &eventResponse, nil +} + +func GetJsonFileEvents(authData AuthData, ffsURI string, query Query, pgToken *string) (*[]JsonFileEvent, *string, error) { + var jsonFileEvents []JsonFileEvent + + if pgToken != nil && *pgToken != "" { + query.PgToken = pgToken + } + + eventQuery, err := ExecQuery(authData, ffsURI, query) + + if err != nil { + return nil, nil, err + } + + fileEventResponse, err := GetJsonFileEventResponse(eventQuery) + + if err != nil { + return nil, nil, err + } + + if fileEventResponse.Problems != nil { + problems, err := json.Marshal(fileEventResponse.Problems) + + if err != nil { + return nil, nil, err + } + + return nil, nil, errors.New(string(problems)) + } + + if len(fileEventResponse.FileEvents) == 0 { + fileEventResponse.FileEvents = nil + } else { + jsonFileEvents = append(jsonFileEvents, fileEventResponse.FileEvents...) + } + + var nextPgToken *string + + if fileEventResponse.NextPgToken != nil && *fileEventResponse.NextPgToken != "" { + nextPgToken = fileEventResponse.NextPgToken + } + + var nextJsonFileEvents *[]JsonFileEvent + + for { + if nextPgToken == nil || *nextPgToken == "" { + break + } else { + nextJsonFileEvents, nextPgToken, err = GetJsonFileEvents(authData, ffsURI, query, pgToken) + + if err != nil { + return nil, nil, err + } + + if nextJsonFileEvents != nil && len(*nextJsonFileEvents) != 0 { + jsonFileEvents = append(jsonFileEvents, *nextJsonFileEvents...) + } + + nextJsonFileEvents = nil + } + } + + return &jsonFileEvents, nil, nil +}