Skip to content

Commit 8ccb76b

Browse files
sklavasklava
committed
Merged in feature/SCFF-49 (pull request #18)
Feature/SCFF-49
2 parents a45d5f7 + 6b35311 commit 8ccb76b

5 files changed

Lines changed: 96 additions & 22 deletions

File tree

‎main.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ var (
3434
sumoCategory = kingpin.Flag("sumo-category", "Sumo Logic Category").Default("").OverrideDefaultFromEnvar("SUMO_CATEGORY").String()
3535
sumoName = kingpin.Flag("sumo-name", "Sumo Logic Name").Default("").OverrideDefaultFromEnvar("SUMO_NAME").String()
3636
sumoHost = kingpin.Flag("sumo-host", "Sumo Logic Host").Default("").OverrideDefaultFromEnvar("SUMO_HOST").String()
37+
verboseLogMessages = kingpin.Flag("verbose-log-messages", "Verbose Log Messages").Default("true").OverrideDefaultFromEnvar("VERBOSE_LOG_MESSAGES").Bool()
3738
)
3839

3940
var (
@@ -68,6 +69,7 @@ func main() {
6869
if *sumoCategory != "" {
6970
logging.Info.Println("Sumo Logic Category: " + *sumoCategory)
7071
}
72+
logging.Info.Printf("Verbose Log Messages: %v\n", *verboseLogMessages)
7173
logging.Info.Println("Starting Sumo Logic Nozzle " + version)
7274

7375
c := cfclient.Config{
@@ -88,7 +90,7 @@ func main() {
8890

8991
logging.Info.Println("Creating queue")
9092
queue := eventQueue.NewQueue(make([]*events.Event, 100))
91-
loggingClientSumo := sumoCFFirehose.NewSumoLogicAppender(*sumoEndpoint, 5000, &queue, *eventsBatchSize, *sumoPostMinimumDelay, *sumoCategory, *sumoName, *sumoHost)
93+
loggingClientSumo := sumoCFFirehose.NewSumoLogicAppender(*sumoEndpoint, 5000, &queue, *eventsBatchSize, *sumoPostMinimumDelay, *sumoCategory, *sumoName, *sumoHost, *verboseLogMessages)
9294
go loggingClientSumo.Start() //multi
9395

9496
logging.Info.Println("Creating Events")

‎sumoCFFirehose/sumoLogicAppender.go‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"bytes"
55
"compress/gzip"
66
"errors"
7-
"fmt"
87
"net/http"
98
"runtime"
109
"time"
@@ -27,6 +26,7 @@ type SumoLogicAppender struct {
2726
sumoCategory string
2827
sumoName string
2928
sumoHost string
29+
verboseLogMessages bool
3030
}
3131

3232
type SumoBuffer struct {
@@ -35,7 +35,7 @@ type SumoBuffer struct {
3535
timerIdlebuffer time.Time
3636
}
3737

38-
func NewSumoLogicAppender(urlValue string, connectionTimeoutValue int, nozzleQueue *eventQueue.Queue, eventsBatchSize int, sumoPostMinimumDelay time.Duration, sumoCategory string, sumoName string, sumoHost string) *SumoLogicAppender {
38+
func NewSumoLogicAppender(urlValue string, connectionTimeoutValue int, nozzleQueue *eventQueue.Queue, eventsBatchSize int, sumoPostMinimumDelay time.Duration, sumoCategory string, sumoName string, sumoHost string, verboseLogMessages bool) *SumoLogicAppender {
3939
return &SumoLogicAppender{
4040
url: urlValue,
4141
connectionTimeout: connectionTimeoutValue,
@@ -46,6 +46,7 @@ func NewSumoLogicAppender(urlValue string, connectionTimeoutValue int, nozzleQue
4646
sumoCategory: sumoCategory,
4747
sumoName: sumoName,
4848
sumoHost: sumoHost,
49+
verboseLogMessages: verboseLogMessages,
4950
}
5051
}
5152

@@ -106,7 +107,7 @@ func (s *SumoLogicAppender) Start() {
106107

107108
}
108109

109-
func StringBuilder(event *events.Event) string {
110+
func StringBuilder(event *events.Event, verboseLogMessages bool) string {
110111

111112
eventType := event.Type
112113
var msg []byte
@@ -137,9 +138,24 @@ func StringBuilder(event *events.Event) string {
137138
case "LogMessage":
138139
timestamp := time.Unix(0, event.Fields["timestamp"].(int64)*int64(time.Nanosecond)).String()
139140
event.Fields["timestamp"] = timestamp
140-
message, err := json.Marshal(event)
141-
if err == nil {
142-
msg = message
141+
if verboseLogMessages == true {
142+
message, err := json.Marshal(event)
143+
if err == nil {
144+
msg = message
145+
}
146+
} else {
147+
eventNoVerbose := events.Event{
148+
Fields: map[string]interface{}{
149+
"timestamp": event.Fields["timestamp"],
150+
"cf_app_guid": event.Fields["cf_app_id"],
151+
},
152+
Msg: event.Msg,
153+
Type: event.Type,
154+
}
155+
message, err := json.Marshal(eventNoVerbose)
156+
if err == nil {
157+
msg = message
158+
}
143159
}
144160
case "ValueMetric":
145161
message, err := json.Marshal(event)
@@ -168,7 +184,7 @@ func StringBuilder(event *events.Event) string {
168184
}
169185

170186
func (s *SumoLogicAppender) AppendLogs(buffer *SumoBuffer) {
171-
buffer.logStringToSend.Write([]byte(StringBuilder(s.nozzleQueue.Pop())))
187+
buffer.logStringToSend.Write([]byte(StringBuilder(s.nozzleQueue.Pop(), s.verboseLogMessages)))
172188
buffer.logEventsInCurrentBuffer++
173189

174190
}
@@ -179,7 +195,6 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string /*, wg *sync.WaitG
179195
g := gzip.NewWriter(&buf)
180196
g.Write([]byte(logStringToSend))
181197
g.Close()
182-
fmt.Println(time.Since(s.timerBetweenPost))
183198
for time.Since(s.timerBetweenPost) < s.sumoPostMinimumDelay {
184199
logging.Info. /*Trace*/ Println("Delaying post to honor minimum post delay")
185200
time.Sleep(100 * time.Millisecond)
@@ -199,7 +214,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string /*, wg *sync.WaitG
199214
request.Header.Add("X-Sumo-Host", s.sumoHost)
200215
}
201216
if s.sumoCategory != "" {
202-
request.Header.Add("X-Sumo-Host", s.sumoCategory)
217+
request.Header.Add("X-Sumo-Category", s.sumoCategory)
203218
}
204219

205220
response, err := s.httpClient.Do(request)
@@ -225,7 +240,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string /*, wg *sync.WaitG
225240
request.Header.Add("X-Sumo-Host", s.sumoHost)
226241
}
227242
if s.sumoCategory != "" {
228-
request.Header.Add("X-Sumo-Host", s.sumoCategory)
243+
request.Header.Add("X-Sumo-Category", s.sumoCategory)
229244
}
230245
response, errRetry = s.httpClient.Do(request)
231246
if errRetry != nil {
@@ -240,7 +255,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string /*, wg *sync.WaitG
240255
time.Sleep(300 * time.Millisecond)
241256
return attempt < 5, errRetry
242257
} else if response.StatusCode == 200 {
243-
logging.Info. /*Trace*/ Println("Post of logs successful after retry...")
258+
logging.Trace.Println("Post of logs successful after retry...")
244259
s.timerBetweenPost = time.Now()
245260
statusCode = response.StatusCode
246261
return true, err
@@ -255,7 +270,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string /*, wg *sync.WaitG
255270
logging.Error.Printf("Not able to post after retry, with status code: %d", statusCode)
256271
}
257272
} else if response.StatusCode == 200 {
258-
logging.Info. /*Trace*/ Println("Post of logs successful")
273+
logging.Trace.Println("Post of logs successful")
259274
s.timerBetweenPost = time.Now()
260275
}
261276

‎sumoCFFirehose/sumoLogicAppender_test.go‎

Lines changed: 49 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,57 @@ func testAppenderStringBuilder(t *testing.T) {
4646

4747
finalString := ""
4848
for queue.GetCount() > 0 {
49-
finalString = finalString + StringBuilder(queue.Pop())
49+
finalString = finalString + StringBuilder(queue.Pop(), true)
5050
}
5151
assert.Equal(t, finalString, "2016-12-12 16:02:41.828366387 -0300 CLST"+"\t"+"OUT"+"\t"+"index [01]"+"\n"+
5252
"2016-12-12 16:02:42.844737993 -0300 CLST"+"\t"+"OUT"+"\t"+"index [02]"+"\n"+
5353
"2016-12-12 16:02:43.862436654 -0300 CLST"+"\t"+"OUT"+"\t"+"index [03]"+"\n", "")
5454
}
55+
56+
func testStringBuilderVerboseLogsFalse(t *testing.T) {
57+
eventVerboseLogMessage := Event{
58+
Fields: map[string]interface{}{
59+
"message_type": "OUT",
60+
"source_instance": "0",
61+
"deployment": "cf",
62+
"ip": "10.193.166.47",
63+
"job": "diego_cell",
64+
"job_index": "c62aebe5-16b8-43f5-a589-1267e09b9537",
65+
"cf_ignored_app": "false",
66+
"timestamp": "1483629662001580713",
67+
"source_type": "APP",
68+
"origin": "rep",
69+
"cf_app_id": "7833dc75-4484-409c-9b74-90b6454906c6",
70+
},
71+
Msg: "Triggering 'app usage events fetcher'",
72+
Type: "LogMessage",
73+
}
74+
finalMessage := StringBuilder(&eventVerboseLogMessage, false)
75+
76+
assert.False(t, assert.Contains(t, finalMessage, "source_type", ""), "should be false")
77+
78+
}
79+
80+
func testStringBuilderVerboseLogsTrue(t *testing.T) {
81+
eventVerboseLogMessage := Event{
82+
Fields: map[string]interface{}{
83+
"message_type": "OUT",
84+
"source_instance": "0",
85+
"deployment": "cf",
86+
"ip": "10.193.166.47",
87+
"job": "diego_cell",
88+
"job_index": "c62aebe5-16b8-43f5-a589-1267e09b9537",
89+
"cf_ignored_app": "false",
90+
"timestamp": "1483629662001580713",
91+
"source_type": "APP",
92+
"origin": "rep",
93+
"cf_app_id": "7833dc75-4484-409c-9b74-90b6454906c6",
94+
},
95+
Msg: "Triggering 'app usage events fetcher'",
96+
Type: "LogMessage",
97+
}
98+
finalMessage := StringBuilder(&eventVerboseLogMessage, true)
99+
100+
assert.True(t, assert.Contains(t, finalMessage, "source_type", ""), "should be true")
101+
102+
}

‎tile-history.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,8 @@ history:
1515
- 0.0.13
1616
- 0.0.14
1717
- 0.0.15
18-
version: 0.0.16
18+
- 0.0.16
19+
- 0.0.17
20+
- 0.0.18
21+
- 0.0.19
22+
version: 0.0.20

‎tile.yml‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,12 @@ forms:
4646
- name: sumo-endpoint
4747
type: string
4848
label: Sumo Logic Endpoint
49-
description: Sumo Logic Endpoint to send logs to
49+
description: Sumo Logic Endpoint where to send the logs
50+
- name: sumo-post-minimum-delay
51+
type: string
52+
label: Sumo Logic Post Minimum Delay
53+
description: Delay between post to Sumo Logic
54+
default: 200ms
5055
- name: sumo-category
5156
type: string
5257
label: Sumo Logic Category
@@ -60,7 +65,7 @@ forms:
6065
- name: sumo-host
6166
type: string
6267
label: Sumo Logic Host
63-
description: Sumo Logic Host
68+
description: Sumo Logic Host (this will override the default in Sumo Logic)
6469
optional: true
6570

6671
- name: cf-config
@@ -84,11 +89,6 @@ forms:
8489
label: Log Events Batch Size
8590
description: Events Batch Size to send to Sumo Logic
8691
default: 200
87-
- name: sumo-post-minimum-delay
88-
type: string
89-
label: Sumo Logic Post Minimum Delay
90-
description: Delay between post to Sumo Logic
91-
default: 200ms
9292
- name: events
9393
type: string
9494
label: Comma separated list of events you would like (Default is "LogMessage")
@@ -105,3 +105,8 @@ forms:
105105
default: true
106106
description: Handle if the Nozzle logs will be ignored and not send to Sumo Logic
107107
configurable: false
108+
- name: verbose-log-messages
109+
type: boolean
110+
label: Verbose in LogMessage event
111+
default: true
112+
description: If is false, it will send only the LogMessage and the App id

0 commit comments

Comments
 (0)