Skip to content

Commit f0c9228

Browse files
nmillalafnmillalaf
committed
Merge branch 'hotfix/readme'
2 parents 022c727 + 2bde2c7 commit f0c9228

4 files changed

Lines changed: 58 additions & 60 deletions

File tree

‎README.md‎

Lines changed: 22 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -7,32 +7,26 @@ This Nozzle aggregates all the events from the _Firehose_ feature in Cloud Found
77
```
88
usage: main [<flags>]
99
10-
Flags:
11-
--help Show context-sensitive help (also try --help-long and --help-man).
12-
--api-endpoint=API-ENDPOINT CF API Endpoint
13-
--sumo-endpoint=SUMO-ENDPOINT Sumo Logic Endpoint
14-
--subscription-id="firehose" Cloud Foundry ID for the subscription.
15-
--cloudfoundry-user=CLOUDFOUNDRY-USER
16-
Cloud Foundry User
17-
--cloudfoundry-password=CLOUDFOUNDRY-PASSWORD
18-
Cloud Foundry Password
19-
--events="LogMessage" Comma separated list of events you would like. Valid options are ContainerMetric,
20-
CounterEvent, Error, HttpStart, HttpStartStop, HttpStop, LogMessage, ValueMetric
21-
--nozzle-polling-period=15s Nozzle Polling Period
22-
--log-events-batch-size=200 Log Events Batch Size to send to Sumo
23-
--sumo-post-minimum-delay=200ms
24-
Sumo Logic HTTP Post Minimum Delay
25-
--sumo-category="" Sumo Logic Category
26-
--sumo-name="" Sumo Logic Name
27-
--sumo-host="" Sumo Logic Host
28-
--verbose-log-messages Allow Verbose Log Messages
29-
--custom-metadata="" Use this flag for addingCustom Metadata (key1:value1,key2:value2, etc...)
30-
--include-only-matching-filter=""
31-
Adds an 'Include only' filter to Events content (key1:value1,key2:value2, etc...)
32-
--exclude-always-matching-filter=""
33-
Adds an 'Exclude always' filter to Events content (key1:value1,key2:value2,
34-
etc...)
35-
--version Show application version.
10+
Flags: (See run command, in this document, for syntax of flags)
11+
--help Show context-sensitive help (also try --help-long and --help-man).
12+
--api-endpoint= CF API Endpoint
13+
--sumo-endpoint= SUMO-ENDPOINT Complete URL for the endpoint, copied from the Sumo Logic HTTP Source configuration
14+
--subscription-id="firehose" Cloud Foundry ID for the subscription.
15+
--cloudfoundry-user= Cloud Foundry User
16+
--cloudfoundry-password= Cloud Foundry Password
17+
--events="LogMessage" Comma separated list of events you would like. Valid options are ContainerMetric, CounterEvent, Error, HttpStart, HttpStartStop,
18+
HttpStop, LogMessage, ValueMetric
19+
--nozzle-polling-period=15s How frequently this Nozzle polls the CF Firehose for data
20+
--log-events-batch-size=500 When number of messages in the buffer is equal to this flag, send those to Sumo Logic
21+
--sumo-post-minimum-delay=2000ms Minimum time between HTTP POST to Sumo Logic
22+
--sumo-category="" This value overrides the default 'Source Category' associated with the configured Sumo Logic HTTP Source
23+
--sumo-name="" This value overrides the default 'Source Name' associated with the configured Sumo Logic HTTP Source
24+
--sumo-host="" This value overrides the default 'Source Host' associated with the configured Sumo Logic HTTP Source
25+
--verbose-log-messages Enable Verbose in 'LogMessage' Event. If this flag is NOT present, the LogMessage will contain ONLY the fields: tiemstamp, cf_app_guid, Msg
26+
--custom-metadata="" Use this flag for addingCustom Metadata to the JSON (key1:value1,key2:value2, etc...)
27+
--include-only-matching-filter="" Adds an 'Include only' filter to Events content (key1:value1,key2:value2, etc...)
28+
--exclude-always-matching-filter="" Adds an 'Exclude always' filter to Events content (key1:value1,key2:value2, etc...)
29+
--version Show application version.
3630
```
3731

3832

@@ -43,8 +37,9 @@ There are two ways to run this Nozzle:
4337

4438
### Run as standalone app
4539

40+
This is an example for running the Nozzle using the flags options described above:
4641
```
47-
godep go run main.go --sumo-endpoint=https://sumo-endpoint --api-endpoint=https://api.endpoint --cloudfoundry-user=some_user --cloudfoundry-password=some_password --sumo-post-minimum-delay=200ms --custom-metadata=Key1:Value1,Key2:Value2,Key3:Value3 --log-events-batch-size=200 --events=LogMessage, ValueMetric --verbose-log-messages
42+
godep go run main.go --sumo-endpoint=https://sumo-endpoint --api-endpoint=https://api.endpoint --cloudfoundry-user=some_user --cloudfoundry-password=some_password --sumo-post-minimum-delay=200ms --sumo-host=123.123.123.0 --sumo-category=categoryTest --sumo-name=NameTestMETA --log-events-batch-size=200 --events=LogMessage, ValueMetric --verbose-log-messages
4843
```
4944

5045
If everything goes right, you should see in your terminal the _Nozzle's Logs_ and, in the __Sumo Logic endpoint__ (defined in the _--sumo-endpoint_ flag) you should see the logs according the events you choose (_'LogMessage'_ and _'ValueMetric'_ with _verbose_ in this case).

‎main.go‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,23 +18,23 @@ import (
1818
)
1919

2020
var (
21-
apiEndpoint = kingpin.Flag("api-endpoint", "CF API Endpoint").OverrideDefaultFromEnvar("API_ENDPOINT").String()
22-
sumoEndpoint = kingpin.Flag("sumo-endpoint", "Sumo Logic Endpoint").OverrideDefaultFromEnvar("SUMO_ENDPOINT").String()
21+
apiEndpoint = kingpin.Flag("api-endpoint", "URL to CF API Endpoint").OverrideDefaultFromEnvar("API_ENDPOINT").String()
22+
sumoEndpoint = kingpin.Flag("sumo-endpoint", "SUMO-ENDPOINT Complete URL for the endpoint, copied from the Sumo Logic HTTP Source configuration").OverrideDefaultFromEnvar("SUMO_ENDPOINT").String()
2323
//dopplerEndpoint = kingpin.Flag("doppler-endpoint", "Overwrite default doppler endpoint return by /v2/info").OverrideDefaultFromEnvar("DOPPLER_ENDPOINT").String()
2424
subscriptionId = kingpin.Flag("subscription-id", "Cloud Foundry ID for the subscription.").Default("firehose").OverrideDefaultFromEnvar("FIREHOSE_SUBSCRIPTION_ID").String()
25-
user = kingpin.Flag("cloudfoundry-user", "Cloud Foundry User").OverrideDefaultFromEnvar("CLOUDFOUNDRY_USER").String() //user created in CF, authorized to connect the firehose
26-
password = kingpin.Flag("cloudfoundry-password", "Cloud Foundry Password").OverrideDefaultFromEnvar("CLOUDFOUNDRY_PASSWORD").String() // password created along with the firehose_user //kingpin.Flag("skip-ssl-validation", "Please don't").Default("false").OverrideDefaultFromEnvar("SKIP_SSL_VALIDATION").Bool()
27-
keepAlive, errK = time.ParseDuration("25s") //default Error, ContainerMetric, HttpStart, HttpStop, HttpStartStop, LogMessage, ValueMetric, CounterEvent
25+
user = kingpin.Flag("cloudfoundry-user", "Cloud Foundry User").OverrideDefaultFromEnvar("CLOUDFOUNDRY_USER").String() //user created in CF, authorized to connect the firehose
26+
password = kingpin.Flag("cloudfoundry-password", "Cloud Foundry Password").OverrideDefaultFromEnvar("CLOUDFOUNDRY_PASSWORD").String()
27+
keepAlive, errK = time.ParseDuration("25s") //default Error,ContainerMetric,HttpStart,HttpStop,HttpStartStop,LogMessage,ValueMetric,CounterEvent
2828
wantedEvents = kingpin.Flag("events", fmt.Sprintf("Comma separated list of events you would like. Valid options are %s", eventRouting.GetListAuthorizedEventEvents())).Default("LogMessage").OverrideDefaultFromEnvar("EVENTS").String()
2929
boltDatabasePath = "event.db"
30-
tickerTime = kingpin.Flag("nozzle-polling-period", "Nozzle Polling Period").Default("15s").OverrideDefaultFromEnvar("NOZZLE_POLLING_PERIOD").Duration()
31-
eventsBatchSize = kingpin.Flag("log-events-batch-size", "Log Events Batch Size to send to Sumo").Default("200").OverrideDefaultFromEnvar("LOG_EVENTS_BATCH_SIZE").Int()
32-
sumoPostMinimumDelay = kingpin.Flag("sumo-post-minimum-delay", "Sumo Logic HTTP Post Minimum Delay").Default("200ms").OverrideDefaultFromEnvar("SUMO_POST_MINIMUM_DELAY").Duration()
33-
sumoCategory = kingpin.Flag("sumo-category", "Sumo Logic Category").Default("").OverrideDefaultFromEnvar("SUMO_CATEGORY").String()
34-
sumoName = kingpin.Flag("sumo-name", "Sumo Logic Name").Default("").OverrideDefaultFromEnvar("SUMO_NAME").String()
35-
sumoHost = kingpin.Flag("sumo-host", "Sumo Logic Host").Default("").OverrideDefaultFromEnvar("SUMO_HOST").String()
36-
verboseLogMessages = kingpin.Flag("verbose-log-messages", "Allow Verbose Log Messages").Default("false").OverrideDefaultFromEnvar("VERBOSE_LOG_MESSAGES").Bool()
37-
customMetadata = kingpin.Flag("custom-metadata", "Use this flag for addingCustom Metadata (key1:value1,key2:value2, etc...)").Default("").OverrideDefaultFromEnvar("CUSTOM_METADATA").String()
30+
tickerTime = kingpin.Flag("nozzle-polling-period", "How frequently this Nozzle polls the CF Firehose for data").Default("15s").OverrideDefaultFromEnvar("NOZZLE_POLLING_PERIOD").Duration()
31+
eventsBatchSize = kingpin.Flag("log-events-batch-size", "When number of messages in the buffer is equal to this flag, send those to Sumo Logic").Default("500").OverrideDefaultFromEnvar("LOG_EVENTS_BATCH_SIZE").Int()
32+
sumoPostMinimumDelay = kingpin.Flag("sumo-post-minimum-delay", "Minimum time between HTTP POST to Sumo Logic").Default("2000ms").OverrideDefaultFromEnvar("SUMO_POST_MINIMUM_DELAY").Duration()
33+
sumoCategory = kingpin.Flag("sumo-category", "This value overrides the default 'Source Category' associated with the configured Sumo Logic HTTP Source").Default("").OverrideDefaultFromEnvar("SUMO_CATEGORY").String()
34+
sumoName = kingpin.Flag("sumo-name", "This value overrides the default 'Source Name' associated with the configured Sumo Logic HTTP Source").Default("").OverrideDefaultFromEnvar("SUMO_NAME").String()
35+
sumoHost = kingpin.Flag("sumo-host", "This value overrides the default 'Source Host' associated with the configured Sumo Logic HTTP Source").Default("").OverrideDefaultFromEnvar("SUMO_HOST").String()
36+
verboseLogMessages = kingpin.Flag("verbose-log-messages", "Enable Verbose in 'LogMessage' Event. If this flag NOT present, the LogMessage will contain ONLY the fields: tiemstamp, cf_app_guid, Msg").Default("false").OverrideDefaultFromEnvar("VERBOSE_LOG_MESSAGES").Bool()
37+
customMetadata = kingpin.Flag("custom-metadata", "Use this flag for addingCustom Metadata to the JSON (key1:value1,key2:value2, etc...)").Default("").OverrideDefaultFromEnvar("CUSTOM_METADATA").String()
3838
includeOnlyMatchingFilter = kingpin.Flag("include-only-matching-filter", "Adds an 'Include only' filter to Events content (key1:value1,key2:value2, etc...)").Default("").OverrideDefaultFromEnvar("INCLUDE_ONLY_MATCHING_FILTER").String()
3939
excludeAlwaysMatchingFilter = kingpin.Flag("exclude-always-matching-filter", "Adds an 'Exclude always' filter to Events content (key1:value1,key2:value2, etc...)").Default("").OverrideDefaultFromEnvar("EXCLUDE_ALWAYS_MATCHING_FILTER").String()
4040
)

‎sumoCFFirehose/sumoLogicAppender.go‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"compress/gzip"
66
"encoding/json"
77
"errors"
8+
"fmt"
89
"net/http"
910
"strings"
1011
"time"
@@ -148,7 +149,13 @@ func WantedEvent(event string, includeOnlyMatchingFilter string, excludeAlwaysMa
148149

149150
}
150151

151-
func StringBuilder(event *events.Event, verboseLogMessages bool, includeOnlyMatchingFilter string, excludeAlwaysMatchingFilter string) string {
152+
func StringBuilder(event *events.Event, verboseLogMessages bool, includeOnlyMatchingFilter string, excludeAlwaysMatchingFilter string, customMetadata string) string {
153+
if customMetadata != "" {
154+
customMetadataMap := ParseCustomInput(customMetadata)
155+
for key, value := range customMetadataMap {
156+
event.Fields[key] = value
157+
}
158+
}
152159
eventType := event.Type
153160
var msg []byte
154161
switch eventType {
@@ -192,6 +199,12 @@ func StringBuilder(event *events.Event, verboseLogMessages bool, includeOnlyMatc
192199
Msg: event.Msg,
193200
Type: event.Type,
194201
}
202+
if customMetadata != "" {
203+
customMetadataMap := ParseCustomInput(customMetadata)
204+
for key, value := range customMetadataMap {
205+
eventNoVerbose.Fields[key] = value
206+
}
207+
}
195208
message, err := json.Marshal(eventNoVerbose)
196209
if err == nil {
197210
msg = message
@@ -230,7 +243,7 @@ func StringBuilder(event *events.Event, verboseLogMessages bool, includeOnlyMatc
230243
}
231244

232245
func (s *SumoLogicAppender) AppendLogs(buffer *SumoBuffer) {
233-
buffer.logStringToSend.Write([]byte(StringBuilder(s.nozzleQueue.Pop(), s.verboseLogMessages, s.includeOnlyMatchingFilter, s.excludeAlwaysMatchingFilter)))
246+
buffer.logStringToSend.Write([]byte(StringBuilder(s.nozzleQueue.Pop(), s.verboseLogMessages, s.includeOnlyMatchingFilter, s.excludeAlwaysMatchingFilter, s.customMetadata)))
234247
buffer.logEventsInCurrentBuffer++
235248

236249
}
@@ -244,6 +257,7 @@ func ParseCustomInput(customInput string) map[string]string {
244257
}
245258

246259
func (s *SumoLogicAppender) SendToSumo(logStringToSend string) {
260+
/*REMOVE*/ fmt.Println(logStringToSend)
247261
if logStringToSend != "" {
248262
var buf bytes.Buffer
249263
g := gzip.NewWriter(&buf)
@@ -266,13 +280,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string) {
266280
if s.sumoCategory != "" {
267281
request.Header.Add("X-Sumo-Category", s.sumoCategory)
268282
}
269-
270-
if s.customMetadata != "" {
271-
customMetadataMap := ParseCustomInput(s.customMetadata)
272-
for key, value := range customMetadataMap {
273-
request.Header.Add(key, value)
274-
}
275-
}
283+
/*REMOVE*/ fmt.Println(time.Since(s.timerBetweenPost))
276284
//checking the timer before first POST intent
277285
for time.Since(s.timerBetweenPost) < s.sumoPostMinimumDelay {
278286
logging.Trace.Println("Delaying Post because minimum post timer not expired")
@@ -303,12 +311,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string) {
303311
if s.sumoCategory != "" {
304312
request.Header.Add("X-Sumo-Category", s.sumoCategory)
305313
}
306-
if s.customMetadata != "" {
307-
customMetadataMap := ParseCustomInput(s.customMetadata)
308-
for key, value := range customMetadataMap {
309-
request.Header.Add(key, value)
310-
}
311-
}
314+
312315
//checking the timer before POST (retry intent)
313316
for time.Since(s.timerBetweenPost) < s.sumoPostMinimumDelay {
314317
logging.Trace.Println("Delaying Post because minimum post timer not expired")
@@ -327,7 +330,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string) {
327330
time.Sleep(300 * time.Millisecond)
328331
return attempt < 5, errRetry
329332
} else if response.StatusCode == 200 {
330-
logging.Trace.Println("Post of logs successful after retry...")
333+
logging.Info. /*Trace.*/ Println("Post of logs successful after retry...")
331334
s.timerBetweenPost = time.Now()
332335
statusCode = response.StatusCode
333336
return true, err
@@ -342,7 +345,7 @@ func (s *SumoLogicAppender) SendToSumo(logStringToSend string) {
342345
logging.Error.Printf("Not able to post after retry, with status code: %d", statusCode)
343346
}
344347
} else if response.StatusCode == 200 {
345-
logging.Trace.Println("Post of logs successful")
348+
logging.Info. /*Trace.*/ Println("Post of logs successful")
346349
s.timerBetweenPost = time.Now()
347350
}
348351

‎sumoCFFirehose/sumoLogicAppender_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ func TestAppenderStringBuilder(t *testing.T) {
6565

6666
finalString := ""
6767
for queue.GetCount() > 0 {
68-
finalString = finalString + StringBuilder(queue.Pop(), true, "", "")
68+
finalString = finalString + StringBuilder(queue.Pop(), true, "", "", "")
6969
}
7070
assert.Equal(t, finalString, "{\"Fields\":{\"deployment\":\"cf\",\"ip\":\"10.193.166.33\",\"job\":\"cloud_controller\",\"job_index\":\"c82feee9-2159-4b05-b669-a9929eb59017\",\"name\":\"requests.completed\",\"origin\":\"cc\",\"unit\":\"counter\",\"value\":558108},\"Msg\":\"\",\"Type\":\"ValueMetric\"}\n"+
7171
"{\"Fields\":{\"delta\":9,\"deployment\":\"cf-redis\",\"ip\":\"10.193.166.84\",\"job\":\"dedicated-node\",\"job_index\":\"8081eca4-9e27-49cb-83ce-948e703c0939\",\"name\":\"dropsondeMarshaller.sentEnvelopes\",\"origin\":\"MetronAgent\",\"total\":10249446},\"Msg\":\"\",\"Type\":\"CounterEvent\"}\n"+
@@ -91,7 +91,7 @@ func TestStringBuilderVerboseLogsFalse(t *testing.T) {
9191
Type: "LogMessage",
9292
}
9393

94-
finalMessage := StringBuilder(&eventVerboseLogMessage, false, "", "")
94+
finalMessage := StringBuilder(&eventVerboseLogMessage, false, "", "", "")
9595
assert.NotContains(t, finalMessage, "source_type", "dsds")
9696

9797
}
@@ -114,7 +114,7 @@ func TestStringBuilderVerboseLogsTrue(t *testing.T) {
114114
Msg: "Triggering 'app usage events fetcher'",
115115
Type: "LogMessage",
116116
}
117-
finalMessage := StringBuilder(&eventVerboseLogMessage, true, "", "")
117+
finalMessage := StringBuilder(&eventVerboseLogMessage, true, "", "", "")
118118

119119
assert.Contains(t, finalMessage, "source_type", "")
120120

0 commit comments

Comments
 (0)