SCFF-49 implemented flag verboseLogMessages for LogMessage Event

Author: nmillalaf

main.go

@@ -34,6 +34,7 @@ var (
 	sumoCategory         = kingpin.Flag("sumo-category", "Sumo Logic Category").Default("").OverrideDefaultFromEnvar("SUMO_CATEGORY").String()
 	sumoName             = kingpin.Flag("sumo-name", "Sumo Logic Name").Default("").OverrideDefaultFromEnvar("SUMO_NAME").String()
 	sumoHost             = kingpin.Flag("sumo-host", "Sumo Logic Host").Default("").OverrideDefaultFromEnvar("SUMO_HOST").String()
+	verboseLogMessages   = kingpin.Flag("verbose-log-messages", "Verbose Log Messages").Default("false").OverrideDefaultFromEnvar("VERBOSE_LOG_MESSAGES").Bool()
 )
 
 var (
@@ -68,6 +69,7 @@ func main() {
 	if *sumoCategory != "" {
 		logging.Info.Println("Sumo Logic Category: " + *sumoCategory)
 	}
+	logging.Info.Printf("Verbose Log Messages: %v\n", *verboseLogMessages)
 	logging.Info.Println("Starting Sumo Logic Nozzle " + version)
 
 	c := cfclient.Config{
@@ -88,7 +90,7 @@ func main() {
 
 	logging.Info.Println("Creating queue")
 	queue := eventQueue.NewQueue(make([]*events.Event, 100))
-	loggingClientSumo := sumoCFFirehose.NewSumoLogicAppender(*sumoEndpoint, 5000, &queue, *eventsBatchSize, *sumoPostMinimumDelay, *sumoCategory, *sumoName, *sumoHost)
+	loggingClientSumo := sumoCFFirehose.NewSumoLogicAppender(*sumoEndpoint, 5000, &queue, *eventsBatchSize, *sumoPostMinimumDelay, *sumoCategory, *sumoName, *sumoHost, *verboseLogMessages)
 	go loggingClientSumo.Start() //multi
 
 	logging.Info.Println("Creating Events")

sumoCFFirehose/sumoLogicAppender.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"compress/gzip"
 	"errors"
-	"fmt"
 	"net/http"
 	"runtime"
 	"time"
@@ -27,6 +26,7 @@ type SumoLogicAppender struct {
 	sumoCategory         string
 	sumoName             string
 	sumoHost             string
+	verboseLogMessages   bool
 }
 
 type SumoBuffer struct {
@@ -35,7 +35,7 @@ type SumoBuffer struct {
 	timerIdlebuffer          time.Time
 }
 
-func NewSumoLogicAppender(urlValue string, connectionTimeoutValue int, nozzleQueue *eventQueue.Queue, eventsBatchSize int, sumoPostMinimumDelay time.Duration, sumoCategory string, sumoName string, sumoHost string) *SumoLogicAppender {
+func NewSumoLogicAppender(urlValue string, connectionTimeoutValue int, nozzleQueue *eventQueue.Queue, eventsBatchSize int, sumoPostMinimumDelay time.Duration, sumoCategory string, sumoName string, sumoHost string, verboseLogMessages bool) *SumoLogicAppender {
 	return &SumoLogicAppender{
 		url:                  urlValue,
 		connectionTimeout:    connectionTimeoutValue,
@@ -46,6 +46,7 @@ func NewSumoLogicAppender(urlValue string, connectionTimeoutValue int, nozzleQue
 		sumoCategory:         sumoCategory,
 		sumoName:             sumoName,
 		sumoHost:             sumoHost,
+		verboseLogMessages:   verboseLogMessages,
 	}
 }
 
@@ -106,7 +107,7 @@ func (s *SumoLogicAppender) Start() {
 
 }
 
-func StringBuilder(event *events.Event) string {
+func StringBuilder(event *events.Event, verboseLogMessages bool) string {
 
 	eventType := event.Type
 	var msg []byte
@@ -135,11 +136,25 @@ func StringBuilder(event *events.Event) string {
 			msg = message
 		}
 	case "LogMessage":
-		timestamp := time.Unix(0, event.Fields["timestamp"].(int64)*int64(time.Nanosecond)).String()
-		event.Fields["timestamp"] = timestamp
-		message, err := json.Marshal(event)
-		if err == nil {
-			msg = message
+		if verboseLogMessages == true {
+			timestamp := time.Unix(0, event.Fields["timestamp"].(int64)*int64(time.Nanosecond)).String()
+			event.Fields["timestamp"] = timestamp
+			message, err := json.Marshal(event)
+			if err == nil {
+				msg = message
+			}
+		} else {
+			eventNoVerbose := events.Event{
+				Fields: map[string]interface{}{
+					"cf_app_guid": event.Fields["cf_app_id"],
+				},
+				Msg:  event.Msg,
+				Type: event.Type,
+			}
+			message, err := json.Marshal(eventNoVerbose)
+			if err == nil {
+				msg = message
+			}
 		}
 	case "ValueMetric":
 		message, err := json.Marshal(event)
@@ -168,7 +183,7 @@ func StringBuilder(event *events.Event) string {
 }
 
 func (s *SumoLogicAppender) AppendLogs(buffer *SumoBuffer) {
-	buffer.logStringToSend.Write([]byte(StringBuilder(s.nozzleQueue.Pop())))
+	buffer.logStringToSend.Write([]byte(StringBuilder(s.nozzleQueue.Pop(), s.verboseLogMessages)))
 	buffer.logEventsInCurrentBuffer++
 
 }
@@ -179,7 +194,6 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string /*, wg *sync.WaitG
 		g := gzip.NewWriter(&buf)
 		g.Write([]byte(logStringToSend))
 		g.Close()
-		fmt.Println(time.Since(s.timerBetweenPost))
 		for time.Since(s.timerBetweenPost) < s.sumoPostMinimumDelay {
 			logging.Info. /*Trace*/ Println("Delaying post to honor minimum post delay")
 			time.Sleep(100 * time.Millisecond)
@@ -199,7 +213,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string /*, wg *sync.WaitG
 			request.Header.Add("X-Sumo-Host", s.sumoHost)
 		}
 		if s.sumoCategory != "" {
-			request.Header.Add("X-Sumo-Host", s.sumoCategory)
+			request.Header.Add("X-Sumo-Category", s.sumoCategory)
 		}
 
 		response, err := s.httpClient.Do(request)
@@ -225,7 +239,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string /*, wg *sync.WaitG
 					request.Header.Add("X-Sumo-Host", s.sumoHost)
 				}
 				if s.sumoCategory != "" {
-					request.Header.Add("X-Sumo-Host", s.sumoCategory)
+					request.Header.Add("X-Sumo-Category", s.sumoCategory)
 				}
 				response, errRetry = s.httpClient.Do(request)
 				if errRetry != nil {
@@ -240,7 +254,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string /*, wg *sync.WaitG
 					time.Sleep(300 * time.Millisecond)
 					return attempt < 5, errRetry
 				} else if response.StatusCode == 200 {
-					logging.Info. /*Trace*/ Println("Post of logs successful after retry...")
+					logging.Trace.Println("Post of logs successful after retry...")
 					s.timerBetweenPost = time.Now()
 					statusCode = response.StatusCode
 					return true, err
@@ -255,7 +269,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string /*, wg *sync.WaitG
 				logging.Error.Printf("Not able to post after retry, with status code: %d", statusCode)
 			}
 		} else if response.StatusCode == 200 {
-			logging.Info. /*Trace*/ Println("Post of logs successful")
+			logging.Trace.Println("Post of logs successful")
 			s.timerBetweenPost = time.Now()
 		}
 

sumoCFFirehose/sumoLogicAppender_test.go

@@ -46,9 +46,57 @@ func testAppenderStringBuilder(t *testing.T) {
 
 	finalString := ""
 	for queue.GetCount() > 0 {
-		finalString = finalString + StringBuilder(queue.Pop())
+		finalString = finalString + StringBuilder(queue.Pop(), true)
 	}
 	assert.Equal(t, finalString, "2016-12-12 16:02:41.828366387 -0300 CLST"+"\t"+"OUT"+"\t"+"index [01]"+"\n"+
 		"2016-12-12 16:02:42.844737993 -0300 CLST"+"\t"+"OUT"+"\t"+"index [02]"+"\n"+
 		"2016-12-12 16:02:43.862436654 -0300 CLST"+"\t"+"OUT"+"\t"+"index [03]"+"\n", "")
 }
+
+func testStringBuilderVerboseLogsFalse(t *testing.T) {
+	eventVerboseLogMessage := Event{
+		Fields: map[string]interface{}{
+			"message_type":    "OUT",
+			"source_instance": "0",
+			"deployment":      "cf",
+			"ip":              "10.193.166.47",
+			"job":             "diego_cell",
+			"job_index":       "c62aebe5-16b8-43f5-a589-1267e09b9537",
+			"cf_ignored_app":  "false",
+			"timestamp":       "1483629662001580713",
+			"source_type":     "APP",
+			"origin":          "rep",
+			"cf_app_id":       "7833dc75-4484-409c-9b74-90b6454906c6",
+		},
+		Msg:  "Triggering 'app usage events fetcher'",
+		Type: "LogMessage",
+	}
+	finalMessage := StringBuilder(&eventVerboseLogMessage, false)
+
+	assert.False(t, assert.Contains(t, finalMessage, "timestamp", ""), "should be false")
+
+}
+
+func testStringBuilderVerboseLogsTrue(t *testing.T) {
+	eventVerboseLogMessage := Event{
+		Fields: map[string]interface{}{
+			"message_type":    "OUT",
+			"source_instance": "0",
+			"deployment":      "cf",
+			"ip":              "10.193.166.47",
+			"job":             "diego_cell",
+			"job_index":       "c62aebe5-16b8-43f5-a589-1267e09b9537",
+			"cf_ignored_app":  "false",
+			"timestamp":       "1483629662001580713",
+			"source_type":     "APP",
+			"origin":          "rep",
+			"cf_app_id":       "7833dc75-4484-409c-9b74-90b6454906c6",
+		},
+		Msg:  "Triggering 'app usage events fetcher'",
+		Type: "LogMessage",
+	}
+	finalMessage := StringBuilder(&eventVerboseLogMessage, true)
+
+	assert.True(t, assert.Contains(t, finalMessage, "timestamp", ""), "should be true")
+
+}

tile-history.yml

@@ -15,4 +15,8 @@ history:
 - 0.0.13
 - 0.0.14
 - 0.0.15
-version: 0.0.16
+- 0.0.16
+- 0.0.17
+- 0.0.18
+- 0.0.19
+version: 0.0.20

tile.yml

@@ -105,3 +105,8 @@ forms:
     default: true
     description: Handle if the Nozzle logs will be ignored and not send to Sumo Logic
     configurable: false
+  - name: verbose-log-messages
+    type: boolean
+    label: Verbose in LogMessage event
+    default: true
+    description: If is false, it will send only the LogMessage and the App id