Merge branch 'hotfix/filtering'

Author: nmillalaf

README.md

@@ -1,5 +1,5 @@
 # cloudfoundry-sumologic-nozzle
-
+===============================
 This Nozzle aggregates all the events from the _Firehose_ feature in Cloud Foundry towards Sumo Logic
 
 ### Options of use
@@ -11,6 +11,7 @@ Flags:  (See run command, in this document, for syntax of flags)
 --help                              Show context-sensitive help (also try --help-long and --help-man).
 --api-endpoint=                     CF API Endpoint
 --sumo-endpoint=                    SUMO-ENDPOINT Complete URL for the endpoint, copied from the Sumo Logic HTTP Source configuration
+--doppler-endpoint=                 Overwrite default doppler endpoint return by /v2/info
 --subscription-id="firehose"        Cloud Foundry ID for the subscription.
 --cloudfoundry-user=                Cloud Foundry User
 --cloudfoundry-password=            Cloud Foundry Password
@@ -67,17 +68,18 @@ in this case, Only the events that contains a _**job:diego-cell**_ field OR a _*
 
 * **Case 4**:
 **Include-Only filter**=job:diego_cell,source_type:other
-**Exclude-Always filter**=source_type:other,origin:rep
+**Exclude-Always filter**=source_type:app,origin:rep
 In this case, all the events that contains a _**job:diego-cell**_ field OR a _**source-type:other**_ field will be sent to Sumo Logic
 **AND also**
-All the events that contains a _**source-type:other**_ field OR an _**origin:rep**_ field will be not sent to Sumo Logic
+All the events that contains a _**source-type:other**_ field OR an _**origin:app**_ field will be not sent to Sumo Logic.
+
+**IMPORTANT**: **Exclude filter _overrides_ Include filter** . This way if one or more of the App's logs fields **match both filters** (contains a _Include-Only filter_ field and a _Exclude-Always_ filter), this log will be **NOT** sent to Sumo Logic.
 
-If an event **share both filters** (contains a _Include-Only filter_ field and a _Exclude-Always_ filter field), Only the _**Include-Only**_ filter will be considered.
 
 The correct way of using those flags will be something like this:
 
 ```
-godep go run main.go --sumo-endpoint=https://sumo-endpoint --api-endpoint=https://api.endpoint --skip-ssl-validation --cloudfoundry-user=some_user --cloudfoundry-password=some_password --sumo-post-minimum-delay=200ms --log-events-batch-size=200 --events=LogMessage, ValueMetric   --include-only-matching-filter=job:diego_cell,source_type:app --exclude-always-matching-filter=source_type:other,unit:count 
+godep go run main.go --sumo-endpoint=https://sumo-endpoint --api-endpoint=https://api.endpoint --skip-ssl-validation --cloudfoundry-user=some_user --cloudfoundry-password=some_password --sumo-post-minimum-delay=200ms --log-events-batch-size=200 --events=LogMessage, ValueMetric   --include-only-matching-filter=job:diego_cell,source_type:app --exclude-always-matching-filter=source_type:other,unit:count
 ```
 
 
@@ -89,27 +91,28 @@ The tile configuration is handled in the 'tile.yml' file. (If you want to modify
 
 #### Steps to run as this Nozzle as a tile in PCF:
 
- ##### Step 1 - Install the tile-generator python package
-
+##### Step 1 - Install the tile-generator python package
 * Follow the Official Pivotal Instructions: http://docs.pivotal.io/tiledev/tile-generator.html#how-to
 (only until half of the _step 3_, DON'T DO 'tile init', only cd into the 'cloudfoundry-sumologic-nozzle' folder)
 
- ##### Step 2 - Check the tile file
+##### Step 2 - Check the tile file
+
 * If you want to add more settings to the tile or remove some. Check the Official Pivotal Documentation for more options http://docs.pivotal.io/tiledev/tile-generator.html#define
 
- ##### Step 3 - Prepare your code:
+##### Step 3 - Prepare your code:
+
 * Zip your entire code and place the zip file into the root directory of the project for which you wish to create a tile. For this tile use this command: (you should do this in a new terminal window)
 
     ```
     zip -r sumo-logic-nozzle.zip bitbucket-pipelines.yml caching/ ci/ eventQueue/ eventRouting/ events/ firehoseclient/ glide.yaml glide.lock Godeps/ LICENSE logging/ main.go manifest.yml event.db Procfile sumoCFFirehose/ utils/ vendor/
     ```
- ##### Step 4 - Build tile file
+##### Step 4 - Build tile file
 * go to the 'tile-generator' terminal window and run
 
     ```
     $ tile build
     ```
- ##### Step 4 - Install the tile in Pivotal Cloud Foundry
+##### Step 5 - Install the tile in Pivotal Cloud Foundry
 * Login with proper credentials into the OPS Manager and import the .pivotal file created above and wait.
 * Then add it to the Installation Dashboard to configure it. You should able to configure the settings created in the tile file.
 * Update the changes and you should start to see some logs in the Sumo Logic Endpoint defined.
@@ -134,6 +137,7 @@ $ cf push cloudfoundry-sumologic-nozzle --no-start
 ```
 $ cf set-env cloudfoundry-sumologic-nozzle API_ENDPOINT https://api.[your cf system domain]
 $ cf set-env cloudfoundry-sumologic-nozzle SUMO_ENDPOINT https://sumo-endpoint
+$ cf set-env cloudfoundry-sumologic-nozzle DOPPLER_ENDPOINT: wss://doppler.bosh-lite.com:443
 $ cf set-env cloudfoundry-sumologic-nozzle FIREHOSE_SUBSCRIPTION_ID cloudfoundry-sumologic-nozzle
 $ cf set-env cloudfoundry-sumologic-nozzle CLOUDFOUNDRY_USER [your doppler.firehose enabled user]
 $ cf set-env cloudfoundry-sumologic-nozzle CLOUDFOUNDRY_PASSWORD [your doppler.firehose enabled user password]

main.go

@@ -18,9 +18,9 @@ import (
 )
 
 var (
-	apiEndpoint  = kingpin.Flag("api-endpoint", "URL to CF API Endpoint").OverrideDefaultFromEnvar("API_ENDPOINT").String()
-	sumoEndpoint = kingpin.Flag("sumo-endpoint", "SUMO-ENDPOINT Complete URL for the endpoint, copied from the Sumo Logic HTTP Source configuration").OverrideDefaultFromEnvar("SUMO_ENDPOINT").String()
-	//dopplerEndpoint      = kingpin.Flag("doppler-endpoint", "Overwrite default doppler endpoint return by /v2/info").OverrideDefaultFromEnvar("DOPPLER_ENDPOINT").String()
+	apiEndpoint                 = kingpin.Flag("api-endpoint", "URL to CF API Endpoint").OverrideDefaultFromEnvar("API_ENDPOINT").String()
+	sumoEndpoint                = kingpin.Flag("sumo-endpoint", "SUMO-ENDPOINT Complete URL for the endpoint, copied from the Sumo Logic HTTP Source configuration").OverrideDefaultFromEnvar("SUMO_ENDPOINT").String()
+	dopplerEndpoint             = kingpin.Flag("doppler-endpoint", "Overwrite default doppler endpoint return by /v2/info").OverrideDefaultFromEnvar("DOPPLER_ENDPOINT").String()
 	subscriptionId              = kingpin.Flag("subscription-id", "Cloud Foundry ID for the subscription.").Default("firehose").OverrideDefaultFromEnvar("FIREHOSE_SUBSCRIPTION_ID").String()
 	user                        = kingpin.Flag("cloudfoundry-user", "Cloud Foundry User").OverrideDefaultFromEnvar("CLOUDFOUNDRY_USER").String() //user created in CF, authorized to connect the firehose
 	password                    = kingpin.Flag("cloudfoundry-password", "Cloud Foundry Password").OverrideDefaultFromEnvar("CLOUDFOUNDRY_PASSWORD").String()
@@ -54,7 +54,6 @@ func main() {
 	logging.Info.Println("Set Configurations:")
 	logging.Info.Println("CF API Endpoint: " + *apiEndpoint)
 	logging.Info.Println("Sumo Logic Endpoint: " + *sumoEndpoint)
-	//logging.Info.Println("Cloud foundry Doppler Endpoint: " + *dopplerEndpoint) //TODO
 	logging.Info.Println("Cloud Foundry Nozzle Subscription ID: " + *subscriptionId)
 	logging.Info.Println("Cloud Foundry User: " + *user)
 	logging.Info.Println("Events Selected: " + *wantedEvents)
@@ -86,6 +85,11 @@ func main() {
 		logging.Error.Fatal("Error setting up CF Client: ", errCfClient)
 		os.Exit(1)
 	}
+	if len(*dopplerEndpoint) > 0 {
+		cfClient.Endpoint.DopplerEndpoint = *dopplerEndpoint
+	}
+	logging.Info.Printf("Using %s as doppler endpoint", cfClient.Endpoint.DopplerEndpoint)
+
 	//Creating Caching
 	var cachingClient caching.Caching
 	if caching.IsNeeded(*wantedEvents) {

manifest.yml

@@ -3,10 +3,11 @@ applications:
 env:
   API_ENDPOINT: https://api.some_api
   SUMO_ENDPOINT: https://some_endpoint
+  DOPPLER_ENDPOINT: wss://doppler.bosh-lite.com:443
   FIREHOSE_SUBSCRIPTION_ID: cloudfoundry-sumologic-nozzle
   CLOUDFOUNDRY_USER: firehose_user_authorized
   CLOUDFOUNDRY_PASSWORD: password_firehose_user_authorized
-  EVENTS: LogMessage
+  EVENTS: LogMessage,ValueMetric,Error,ContainerMetric
   SKIP_SSL_VALIDATION: false
   NOZZLE_POLLING_PERIOD: 15s
   LOG_EVENTS_BATCHSIZE: 200

sumoCFFirehose/sumoLogicAppender.go

@@ -119,6 +119,11 @@ func WantedEvent(event string, includeOnlyMatchingFilter string, excludeAlwaysMa
 		subsliceExclude := ParseCustomInput(excludeAlwaysMatchingFilter)
 		for key, value := range subsliceInclude {
 			if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
+				for key, value := range subsliceExclude {
+					if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
+						return false
+					}
+				}
 				return true
 			}
 		}
@@ -127,7 +132,7 @@ func WantedEvent(event string, includeOnlyMatchingFilter string, excludeAlwaysMa
 				return false
 			}
 		}
-		return false
+		return true
 	} else if includeOnlyMatchingFilter != "" {
 		subslice := ParseCustomInput(includeOnlyMatchingFilter)
 		for key, value := range subslice {

sumoCFFirehose/sumoLogicAppender_test.go

@@ -188,7 +188,7 @@ func TestSendExcludeAlwaysFilter(t *testing.T) {
 	excludeAlwaysFilter := "source_type:other,cf_app_id:7833dc75-4484-409c-9b74-90b6454906c6"
 	assert.False(t, WantedEvent(buf.String(), "", excludeAlwaysFilter), "This Event should be excluded")
 }
-func TestSendIncludeOnlyAlwaysFilter(t *testing.T) {
+func TestSendIncludeExcludeFilterOverride(t *testing.T) {
 	eventToExclude := Event{
 		Fields: map[string]interface{}{
 			"message_type":    "OUT",
@@ -215,8 +215,99 @@ func TestSendIncludeOnlyAlwaysFilter(t *testing.T) {
 	buf.Write(msg)
 	includeOnlyFilter := "job:diego_cell,source_type:other"
 	excludeAlwaysFilter := "source_type:other,cf_app_id:7833dc75-4484-409c-9b74-90b6454906c6"
+	assert.False(t, WantedEvent(buf.String(), includeOnlyFilter, excludeAlwaysFilter), "This Event should be not included, override filter")
+}
+
+func TestSendIncludeExcludeFilterAppMatchIncluded(t *testing.T) {
+	eventToExclude := Event{
+		Fields: map[string]interface{}{
+			"message_type":    "OUT",
+			"source_instance": 0,
+			"deployment":      "cf",
+			"ip":              "10.193.166.47",
+			"job":             "diego_cell",
+			"job_index":       "c62aebe5-16b8-43f5-a589-1267e09b9537",
+			"cf_ignored_app":  "false",
+			"timestamp":       "2017-01-10 17:31:02.662133274 -0300 CLST",
+			"source_type":     "APP",
+			"origin":          "rep",
+			"cf_app_id":       "7833dc75-4484-409c-9b74-90b6454906c6",
+		},
+		Msg:  "Triggering 'app usage events fetcher'",
+		Type: "LogMessage",
+	}
+	message, err := json.Marshal(eventToExclude)
+	var msg []byte
+	if err == nil {
+		msg = message
+	}
+	buf := new(bytes.Buffer)
+	buf.Write(msg)
+	includeOnlyFilter := "job:diego_cell,source_type:other"
+	excludeAlwaysFilter := "source_type:other,origin:router"
 	assert.True(t, WantedEvent(buf.String(), includeOnlyFilter, excludeAlwaysFilter), "This Event should be included")
 }
+
+func TestSendIncludeExcludeFilterAppMatchExcluded(t *testing.T) {
+	eventToExclude := Event{
+		Fields: map[string]interface{}{
+			"message_type":    "OUT",
+			"source_instance": 0,
+			"deployment":      "cf",
+			"ip":              "10.193.166.47",
+			"job":             "diego_cell",
+			"job_index":       "c62aebe5-16b8-43f5-a589-1267e09b9537",
+			"cf_ignored_app":  "false",
+			"timestamp":       "2017-01-10 17:31:02.662133274 -0300 CLST",
+			"source_type":     "APP",
+			"origin":          "rep",
+			"cf_app_id":       "7833dc75-4484-409c-9b74-90b6454906c6",
+		},
+		Msg:  "Triggering 'app usage events fetcher'",
+		Type: "LogMessage",
+	}
+	message, err := json.Marshal(eventToExclude)
+	var msg []byte
+	if err == nil {
+		msg = message
+	}
+	buf := new(bytes.Buffer)
+	buf.Write(msg)
+	includeOnlyFilter := "job:dedicated-node,source_type:other"
+	excludeAlwaysFilter := "source_type:other,origin:rep"
+	assert.False(t, WantedEvent(buf.String(), includeOnlyFilter, excludeAlwaysFilter), "This Event should not be included")
+}
+
+func TestSendIncludeExcludeFilterAppNotMatchAnyFilter(t *testing.T) {
+	eventToExclude := Event{
+		Fields: map[string]interface{}{
+			"message_type":    "OUT",
+			"source_instance": 0,
+			"deployment":      "cf",
+			"ip":              "10.193.166.47",
+			"job":             "diego_cell",
+			"job_index":       "c62aebe5-16b8-43f5-a589-1267e09b9537",
+			"cf_ignored_app":  "false",
+			"timestamp":       "2017-01-10 17:31:02.662133274 -0300 CLST",
+			"source_type":     "APP",
+			"origin":          "rep",
+			"cf_app_id":       "7833dc75-4484-409c-9b74-90b6454906c6",
+		},
+		Msg:  "Triggering 'app usage events fetcher'",
+		Type: "LogMessage",
+	}
+	message, err := json.Marshal(eventToExclude)
+	var msg []byte
+	if err == nil {
+		msg = message
+	}
+	buf := new(bytes.Buffer)
+	buf.Write(msg)
+	includeOnlyFilter := "job:dedicated-node,source_type:other"
+	excludeAlwaysFilter := "source_type:other,origin:reps"
+	assert.True(t, WantedEvent(buf.String(), includeOnlyFilter, excludeAlwaysFilter), "This Event should be included")
+}
+
 func TestSendNoFilter(t *testing.T) {
 	eventToExclude := Event{
 		Fields: map[string]interface{}{

tile.yml

@@ -93,6 +93,11 @@ forms:
     type: string
     label: Cloud Foundry API Endpoint
     description: URL to CF API Endpoint
+  - name: doppler-endpoint
+    type: string
+    label: Doppler Endpoint
+    description: Overwrite default doppler endpoint return by /v2/info
+    optional: true
   - name: cloudfoundry-user
     type: string
     label: Cloud Foundry User
@@ -126,9 +131,3 @@ forms:
     label: Nozzle Polling Period
     default: 15s
     description: How frequently this Nozzle polls the CF Firehose for data
-  - name: f2s-disable-logging
-    type: boolean
-    label: Disable Logging (Nozzle's Logs)
-    default: true
-    description: Handle if the Nozzle logs will be ignored and not send to Sumo Logic
-    configurable: false