Merge pull request #406 from splunk/fix-mod-inputs-examples

Fix mod inputs examples

Author: fantavlik

docker-compose.yml

@@ -11,7 +11,9 @@ services:
             - SPLUNK_APPS_URL=https://github.com/splunk/sdk-app-collection/releases/download/v1.1.0/sdkappcollection.tgz
         volumes:
             - ./examples/github_forks:/opt/splunk/etc/apps/github_forks
+            - ./splunklib:/opt/splunk/etc/apps/github_forks/lib/splunklib
             - ./examples/random_numbers:/opt/splunk/etc/apps/random_numbers
+            - ./splunklib:/opt/splunk/etc/apps/random_numbers/lib/splunklib
             - ./examples/searchcommands_app/package:/opt/splunk/etc/apps/searchcommands_app
             - ./splunklib:/opt/splunk/etc/apps/searchcommands_app/lib/splunklib
             - ./examples/twitted/twitted:/opt/splunk/etc/apps/twitted

examples/github_forks/README.md

@@ -0,0 +1,12 @@
+splunk-sdk-python github_forks example
+========================================
+
+This app provides an example of a modular input that generates the number of repository forks according to the Github API based on the owner and repo_name provided by the user during setup of the input.
+
+To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/github_forks` and `/opt/splunk/etc/apps/github_forks/lib/splunklib` within the `splunk` container.
+
+Once the docker container is up and healthy log into the Splunk UI and setup a new `Github Repository Forks` input by visiting this page: http://localhost:8000/en-US/manager/github_forks/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. Enter values for a Github Repository owner and repo_name, for example owner = `splunk` repo_name = `splunk-sdk-python`.
+
+NOTE: If no Github Repository Forks input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs.
+
+Once the input is created you should be able to see an event when running the following search: `source="github_forks://*"` the event should contain fields for `owner` and `repository` matching the values you input during setup and then a `fork_count` field corresponding to the number of forks the repo has according to the Github API.

examples/github_forks/github_forks.py renamed to examples/github_forks/bin/github_forks.py

@@ -15,10 +15,18 @@
 # under the License.
 
 from __future__ import absolute_import
-import sys, urllib2, json
+import os
+import sys
+import json
+# NOTE: splunklib must exist within github_forks/lib/splunklib for this
+# example to run! To run this locally use `SPLUNK_VERSION=latest docker compose up -d`
+# from the root of this repo which mounts this example and the latest splunklib
+# code together at /opt/splunk/etc/apps/github_forks
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib"))
 
 from splunklib.modularinput import *
 from splunklib import six
+from six.moves import http_client
 
 class MyScript(Script):
     """All modular inputs should inherit from the abstract base class Script
@@ -87,11 +95,9 @@ def validate_input(self, validation_definition):
         # Get the values of the parameters, and construct a URL for the Github API
         owner = validation_definition.parameters["owner"]
         repo_name = validation_definition.parameters["repo_name"]
-        repo_url = "https://api.github.com/repos/%s/%s" % (owner, repo_name)
 
-        # Read the response from the Github API, then parse the JSON data into an object
-        response = urllib2.urlopen(repo_url).read()
-        jsondata = json.loads(response)
+        # Call Github to retrieve repo information
+        jsondata = _get_github_repos(owner, repo_name)
 
         # If there is only 1 field in the jsondata object,some kind or error occurred
         # with the Github API.
@@ -125,9 +131,7 @@ def stream_events(self, inputs, ew):
             repo_name = input_item["repo_name"]
 
             # Get the fork count from the Github API
-            repo_url = "https://api.github.com/repos/%s/%s" % (owner, repo_name)
-            response = urllib2.urlopen(repo_url).read()
-            jsondata = json.loads(response)
+            jsondata = _get_github_repos(owner, repo_name)
             fork_count = jsondata["forks_count"]
 
             # Create an Event object, and set its fields
@@ -139,5 +143,20 @@ def stream_events(self, inputs, ew):
             # Tell the EventWriter to write this event
             ew.write_event(event)
 
+
+def _get_github_repos(owner, repo_name):
+    # Read the response from the Github API, then parse the JSON data into an object
+    repo_path = "/repos/%s/%s" % (owner, repo_name)
+    connection = http_client.HTTPSConnection('api.github.com')
+    headers = {
+        'Content-type': 'application/json',
+        'User-Agent': 'splunk-sdk-python',
+    }
+    connection.request('GET', repo_path, headers=headers)
+    response = connection.getresponse()
+    body = response.read().decode()
+    return json.loads(body)
+
+
 if __name__ == "__main__":
     sys.exit(MyScript().run(sys.argv))

examples/random_numbers/README.md

@@ -0,0 +1,12 @@
+splunk-sdk-python random_numbers example
+========================================
+
+This app provides an example of a modular input that generates a random number between the min and max values provided by the user during setup of the input.
+
+To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/random_numbers` and `/opt/splunk/etc/apps/random_numbers/lib/splunklib` within the `splunk` container.
+
+Once the docker container is up and healthy log into the Splunk UI and setup a new `Random Numbers` input by visiting this page: http://localhost:8000/en-US/manager/random_numbers/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. Enter values for the `min` and `max` values which the random number should be generated between.
+
+NOTE: If no Random Numbers input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs.
+
+Once the input is created you should be able to see an event when running the following search: `source="random_numbers://*"` the event should contain a `number` field with a float between the min and max specified when the input was created.

examples/random_numbers/random_numbers.py renamed to examples/random_numbers/bin/random_numbers.py

@@ -17,6 +17,10 @@
 from __future__ import absolute_import
 import random, sys
 import os
+# NOTE: splunklib must exist within random_numbers/lib/splunklib for this
+# example to run! To run this locally use `SPLUNK_VERSION=latest docker compose up -d`
+# from the root of this repo which mounts this example and the latest splunklib
+# code together at /opt/splunk/etc/apps/random_numbers
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib"))
 
 from splunklib.modularinput import *

setup.py

@@ -15,13 +15,9 @@
 # under the License.
 
 from setuptools import setup, Command
-from contextlib import closing
-from subprocess import check_call, STDOUT
 
 import os
 import sys
-import shutil
-import tarfile
 
 import splunklib