diff --git a/monger/handlers/ingest.go b/monger/handlers/ingest.go index 04acce4..3437518 100644 --- a/monger/handlers/ingest.go +++ b/monger/handlers/ingest.go @@ -1,12 +1,9 @@ package handlers import ( - "encoding/json" + "github.com/NumexaHQ/monger/model" "io/ioutil" "net/http" - "time" - - "github.com/NumexaHQ/monger/model" "github.com/sirupsen/logrus" ) @@ -26,15 +23,13 @@ func (h *Handler) IngestLogs(w http.ResponseWriter, r *http.Request) { // if request, build proxy request // if response, build proxy response if r.Header.Get("X-Numexa-Log-Type") == "request" { - var embededRequest *http.Request - err = json.Unmarshal(body, &embededRequest) + err, newReq, requestTime := model.CreateNewRequest(err, body, r) if err != nil { - logrus.Errorf("Error unmarshalling body: %v", err) - http.Error(w, "Error unmarshalling body", http.StatusBadRequest) + logrus.Errorf("Error creating New Request: %v", err) + http.Error(w, "Error creating New Request", http.StatusInternalServerError) return } - - pr, err := model.ProxyRequestBuilderForHTTPRequest(embededRequest, time.Now(), h.AuthDB, r.URL.String(), apiKey) + pr, err := model.ProxyRequestBuilderForHTTPRequest(newReq, requestTime, h.AuthDB, newReq.URL.String(), apiKey) if err != nil { logrus.Errorf("Error building proxy request: %v", err) } @@ -44,15 +39,13 @@ func (h *Handler) IngestLogs(w http.ResponseWriter, r *http.Request) { }() return } else if r.Header.Get("X-Numexa-Log-Type") == "response" { - var embededResponse *http.Response - err = json.Unmarshal(body, &embededResponse) + err, newResponse, initiatedTime, responseTime := model.CreateNewResponse(r, body) if err != nil { - logrus.Errorf("Error unmarshalling body: %v", err) - http.Error(w, "Error unmarshalling body", http.StatusBadRequest) + logrus.Errorf("Error building New Response: %v", err) + http.Error(w, "Error building New Response", http.StatusInternalServerError) return } - - pr, err := model.ProxyResponseBuilderForHTTPResponse(r.Context(), embededResponse, h.AuthDB, time.Now(), 0, time.Now(), apiKey) + pr, err := model.ProxyResponseBuilderForHTTPResponse(r.Context(), newResponse, h.AuthDB, initiatedTime, 0, responseTime, apiKey) if err != nil { logrus.Errorf("Error building proxy response: %v", err) } diff --git a/monger/model/python_sdk_request.go b/monger/model/python_sdk_request.go new file mode 100644 index 0000000..4f62b4e --- /dev/null +++ b/monger/model/python_sdk_request.go @@ -0,0 +1,60 @@ +package model + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/url" + "time" +) + +type PythonSdkRequest struct { + RequestTime string `json:"request_time"` + SourceIp string `json:"source_ip"` + RequestMethod string `json:"request_method"` + RequestUrl string `json:"request_url"` + RequestHeaders map[string]string `json:"request_headers"` + RequestBody map[string]interface{} `json:"request_body"` +} + +// CreateNewRequest Helper function to Create new Request received from python Sdk +func CreateNewRequest(err error, body []byte, r *http.Request) (error, *http.Request, time.Time) { + bodyFromPythonSdk := &PythonSdkRequest{} + err = json.Unmarshal(body, bodyFromPythonSdk) + if err != nil { + return err, r, time.Now() + } + // Creating Body Bytes for New Request + newRequestBodyBytes, err := json.Marshal(bodyFromPythonSdk.RequestBody) + if err != nil { + return err, r, time.Now() + } + // Creating Body From Body Bytes For New Request + newRequestBody := io.NopCloser(bytes.NewBuffer(newRequestBodyBytes)) + + u, err := url.Parse(bodyFromPythonSdk.RequestUrl) + if err != nil { + return err, r, time.Now() + } + requestTimeStamp, err := time.Parse("2006-01-02 15:04:05.000", bodyFromPythonSdk.RequestTime) + if err != nil { + return err, r, time.Now() + } + // Setting up New Request With Existing Request Context + newReq, err := http.NewRequestWithContext(r.Context(), r.Method, u.String(), newRequestBody) + if err != nil { + return err, newReq, time.Now() + } + //Setting Up new Request URl and RemoteAddress + newReq.URL = u + newReq.RemoteAddr = bodyFromPythonSdk.SourceIp + + // Copying Existing Request Headers into New Request + for key, values := range r.Header { + for _, value := range values { + newReq.Header.Add(key, value) + } + } + return nil, newReq, requestTimeStamp +} diff --git a/monger/model/python_sdk_response.go b/monger/model/python_sdk_response.go new file mode 100644 index 0000000..3982336 --- /dev/null +++ b/monger/model/python_sdk_response.go @@ -0,0 +1,66 @@ +package model + +import ( + "bytes" + "encoding/json" + "github.com/NumexaHQ/monger/utils" + "io" + "net/http" + "time" +) + +type PythonSdkResponse struct { + InitiatedTimestamp string `json:"initiated_timestamp"` + ResponseTimestamp string `json:"response_timestamp"` + ResponseStatusCode uint16 `json:"response_status_code"` + ResponseBody map[string]interface{} `json:"response_body"` +} + +// CreateNewResponse Helper function to Create new Response received from python Sdk +func CreateNewResponse(r *http.Request, body []byte) (error, *http.Response, time.Time, time.Time) { + pythonSdkResponse := &PythonSdkResponse{} + err := json.Unmarshal(body, pythonSdkResponse) + if err != nil { + return err, &http.Response{}, time.Now(), time.Now() + } + // Creating Body Bytes for New Response + newResponseBodyBytes, err := json.Marshal(pythonSdkResponse.ResponseBody) + if err != nil { + return err, &http.Response{}, time.Now(), time.Now() + } + contentEncoding := r.Header.Get("Content-Encoding") + initiatedTimeStamp, err := time.Parse("2006-01-02 15:04:05.000", pythonSdkResponse.InitiatedTimestamp) + if err != nil { + return err, &http.Response{}, time.Now(), time.Now() + } + responseTimestamp, err := time.Parse("2006-01-02 15:04:05.000", pythonSdkResponse.ResponseTimestamp) + if err != nil { + return err, &http.Response{}, time.Now(), time.Now() + } + var bodyBytes []byte + if contentEncoding == "br" { + // Encoding Into Brotli + brotliBytes, err := utils.EncodeIntoBrotli(newResponseBodyBytes) + if err != nil { + return err, &http.Response{}, time.Now(), time.Now() + } + bodyBytes = brotliBytes + + } else if contentEncoding == "gzip" { + // Encoding Into Gzip + gzipBytes, err := utils.EncodeIntoGzip(newResponseBodyBytes) + if err != nil { + return err, &http.Response{}, time.Now(), time.Now() + } + bodyBytes = gzipBytes + } else { + bodyBytes = newResponseBodyBytes + } + proxyResponse := &http.Response{ + StatusCode: int(pythonSdkResponse.ResponseStatusCode), + Body: io.NopCloser(bytes.NewBuffer(bodyBytes)), + ContentLength: int64(len(body)), + Header: r.Header, + } + return nil, proxyResponse, initiatedTimeStamp, responseTimestamp +} diff --git a/monger/utils/http.go b/monger/utils/http.go index ad757e8..aad4908 100644 --- a/monger/utils/http.go +++ b/monger/utils/http.go @@ -1,8 +1,12 @@ package utils import ( + "bytes" + "compress/gzip" "encoding/json" "errors" + "fmt" + "github.com/andybalholm/brotli" "io" ) @@ -47,3 +51,31 @@ func ExtractContentFromRequestBody(rb io.ReadCloser) (string, error) { return content, nil } + +// EncodeIntoBrotli helper function to encode the marshal data into brotli format +func EncodeIntoBrotli(newResponseBodyBytes []byte) ([]byte, error) { + brotliBytes := &bytes.Buffer{} + writer := brotli.NewWriter(brotliBytes) + _, err := writer.Write(newResponseBodyBytes) + if err != nil { + return nil, fmt.Errorf("error Encoding into brotli %s", err) + } + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("error while closing brotli Writer %s", err) + } + return brotliBytes.Bytes(), err +} + +// EncodeIntoGzip helper function to encode the marshal data into gzip format +func EncodeIntoGzip(data []byte) ([]byte, error) { + gzipBytes := &bytes.Buffer{} + writer := gzip.NewWriter(gzipBytes) + _, err := writer.Write(data) + if err != nil { + return nil, fmt.Errorf("error Encoding into Gzip %s", err) + } + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("error while closing gzip Writer %s", err) + } + return gzipBytes.Bytes(), nil +}