diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 123eb5045..71ed1e667 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,12 +45,6 @@ jobs: echo password=changed! >> .splunkrc echo scheme=https >> .splunkrc echo version=${{ matrix.splunk }} >> .splunkrc - - name: Create build dir for ExamplesTestCase::test_build_dir_exists test case - run: | - cd ~ - cd /home/runner/work/splunk-sdk-python/splunk-sdk-python/ - python setup.py build - python setup.py dist - name: Install tox run: pip install tox - name: Test Execution diff --git a/.gitignore b/.gitignore index 3df44b515..2346d353a 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,8 @@ MANIFEST coverage_report test.log examples/*/local -examples/*/metadata/local.meta +examples/**/local.meta +examples/**/*.log tests/searchcommands_data/log/ tests/searchcommands_data/output/ examples/searchcommands_app/searchcommand_app.log @@ -24,7 +25,6 @@ Test Results*.html tests/searchcommands/data/app/app.log splunk_sdk.egg-info/ dist/ -examples/searchcommands_app/package/default/commands.conf examples/searchcommands_app/package/lib/splunklib tests/searchcommands/apps/app_with_logging_configuration/*.log *.observed diff --git a/Makefile b/Makefile index a09da530b..233978781 100644 --- a/Makefile +++ b/Makefile @@ -18,16 +18,11 @@ DATE := `date "+%FT%T%z"` CONTAINER_NAME := 'splunk' .PHONY: all -all: build_app test +all: test init: @echo "$(ATTN_COLOR)==> init $(NO_COLOR)" -.PHONY: build_app -build_app: - @echo "$(ATTN_COLOR)==> build_app $(NO_COLOR)" - @python setup.py build dist - .PHONY: docs docs: @echo "$(ATTN_COLOR)==> docs $(NO_COLOR)" diff --git a/docker-compose.yml b/docker-compose.yml index 0527a30bd..6885cfd5f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,14 @@ services: - SPLUNK_HEC_TOKEN=11111111-1111-1111-1111-1111111111113 - SPLUNK_PASSWORD=changed! - SPLUNK_APPS_URL=https://github.com/splunk/sdk-app-collection/releases/download/v1.1.0/sdkappcollection.tgz + volumes: + - ./examples/github_forks:/opt/splunk/etc/apps/github_forks + - ./splunklib:/opt/splunk/etc/apps/github_forks/lib/splunklib + - ./examples/random_numbers:/opt/splunk/etc/apps/random_numbers + - ./splunklib:/opt/splunk/etc/apps/random_numbers/lib/splunklib + - ./examples/searchcommands_app/package:/opt/splunk/etc/apps/searchcommands_app + - ./splunklib:/opt/splunk/etc/apps/searchcommands_app/lib/splunklib + - ./examples/twitted/twitted:/opt/splunk/etc/apps/twitted ports: - 8000:8000 - 8088:8088 diff --git a/examples/github_forks/README.md b/examples/github_forks/README.md new file mode 100644 index 000000000..1a05c862f --- /dev/null +++ b/examples/github_forks/README.md @@ -0,0 +1,12 @@ +splunk-sdk-python github_forks example +======================================== + +This app provides an example of a modular input that generates the number of repository forks according to the Github API based on the owner and repo_name provided by the user during setup of the input. + +To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/github_forks` and `/opt/splunk/etc/apps/github_forks/lib/splunklib` within the `splunk` container. + +Once the docker container is up and healthy log into the Splunk UI and setup a new `Github Repository Forks` input by visiting this page: http://localhost:8000/en-US/manager/github_forks/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. Enter values for a Github Repository owner and repo_name, for example owner = `splunk` repo_name = `splunk-sdk-python`. + +NOTE: If no Github Repository Forks input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs. + +Once the input is created you should be able to see an event when running the following search: `source="github_forks://*"` the event should contain fields for `owner` and `repository` matching the values you input during setup and then a `fork_count` field corresponding to the number of forks the repo has according to the Github API. \ No newline at end of file diff --git a/examples/github_forks/github_forks.py b/examples/github_forks/bin/github_forks.py similarity index 84% rename from examples/github_forks/github_forks.py rename to examples/github_forks/bin/github_forks.py index 2349bd686..5ffa4e409 100755 --- a/examples/github_forks/github_forks.py +++ b/examples/github_forks/bin/github_forks.py @@ -15,10 +15,18 @@ # under the License. from __future__ import absolute_import -import sys, urllib2, json +import os +import sys +import json +# NOTE: splunklib must exist within github_forks/lib/splunklib for this +# example to run! To run this locally use `SPLUNK_VERSION=latest docker compose up -d` +# from the root of this repo which mounts this example and the latest splunklib +# code together at /opt/splunk/etc/apps/github_forks +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) from splunklib.modularinput import * from splunklib import six +from six.moves import http_client class MyScript(Script): """All modular inputs should inherit from the abstract base class Script @@ -87,11 +95,9 @@ def validate_input(self, validation_definition): # Get the values of the parameters, and construct a URL for the Github API owner = validation_definition.parameters["owner"] repo_name = validation_definition.parameters["repo_name"] - repo_url = "https://api.github.com/repos/%s/%s" % (owner, repo_name) - # Read the response from the Github API, then parse the JSON data into an object - response = urllib2.urlopen(repo_url).read() - jsondata = json.loads(response) + # Call Github to retrieve repo information + jsondata = _get_github_repos(owner, repo_name) # If there is only 1 field in the jsondata object,some kind or error occurred # with the Github API. @@ -125,9 +131,7 @@ def stream_events(self, inputs, ew): repo_name = input_item["repo_name"] # Get the fork count from the Github API - repo_url = "https://api.github.com/repos/%s/%s" % (owner, repo_name) - response = urllib2.urlopen(repo_url).read() - jsondata = json.loads(response) + jsondata = _get_github_repos(owner, repo_name) fork_count = jsondata["forks_count"] # Create an Event object, and set its fields @@ -139,5 +143,20 @@ def stream_events(self, inputs, ew): # Tell the EventWriter to write this event ew.write_event(event) + +def _get_github_repos(owner, repo_name): + # Read the response from the Github API, then parse the JSON data into an object + repo_path = "/repos/%s/%s" % (owner, repo_name) + connection = http_client.HTTPSConnection('api.github.com') + headers = { + 'Content-type': 'application/json', + 'User-Agent': 'splunk-sdk-python', + } + connection.request('GET', repo_path, headers=headers) + response = connection.getresponse() + body = response.read().decode() + return json.loads(body) + + if __name__ == "__main__": sys.exit(MyScript().run(sys.argv)) diff --git a/examples/random_numbers/README.md b/examples/random_numbers/README.md new file mode 100644 index 000000000..7ff4069f2 --- /dev/null +++ b/examples/random_numbers/README.md @@ -0,0 +1,12 @@ +splunk-sdk-python random_numbers example +======================================== + +This app provides an example of a modular input that generates a random number between the min and max values provided by the user during setup of the input. + +To run this example locally run `SPLUNK_VERSION=latest docker compose up -d` from the root of this repository which will mount this example alongside the latest version of splunklib within `/opt/splunk/etc/apps/random_numbers` and `/opt/splunk/etc/apps/random_numbers/lib/splunklib` within the `splunk` container. + +Once the docker container is up and healthy log into the Splunk UI and setup a new `Random Numbers` input by visiting this page: http://localhost:8000/en-US/manager/random_numbers/datainputstats and selecting the "Add new..." button next to the Local Inputs > Random Inputs. Enter values for the `min` and `max` values which the random number should be generated between. + +NOTE: If no Random Numbers input appears then the script is likely not running properly, see https://docs.splunk.com/Documentation/SplunkCloud/latest/AdvancedDev/ModInputsDevTools for more details on debugging the modular input using the command line and relevant logs. + +Once the input is created you should be able to see an event when running the following search: `source="random_numbers://*"` the event should contain a `number` field with a float between the min and max specified when the input was created. \ No newline at end of file diff --git a/examples/random_numbers/random_numbers.py b/examples/random_numbers/bin/random_numbers.py similarity index 95% rename from examples/random_numbers/random_numbers.py rename to examples/random_numbers/bin/random_numbers.py index f0727f0dd..b9673db99 100755 --- a/examples/random_numbers/random_numbers.py +++ b/examples/random_numbers/bin/random_numbers.py @@ -17,6 +17,10 @@ from __future__ import absolute_import import random, sys import os +# NOTE: splunklib must exist within random_numbers/lib/splunklib for this +# example to run! To run this locally use `SPLUNK_VERSION=latest docker compose up -d` +# from the root of this repo which mounts this example and the latest splunklib +# code together at /opt/splunk/etc/apps/random_numbers sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) from splunklib.modularinput import * diff --git a/examples/searchcommands_app/Build-App b/examples/searchcommands_app/Build-App deleted file mode 100755 index a5cdd4a15..000000000 --- a/examples/searchcommands_app/Build-App +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/bash-prologue" ${BASH_SOURCE[0]} 'help,clean,debug-client:' 'hcd:' $* || exit $? - -########### -# Arguments -########### - -eval set -- $args - -while [[ $1 != '--' ]] -do - case $1 in - -h|--help) - usage; # does not return - shift 1 - ;; - -c|--clean) - declare -r clean="clean" - shift 1 - ;; - -d|--debug-client) - [[ -f "$d" ]] || error 1 "Debug client '$2' does not exist." - declare -r debugClient="--debug-client '$2'" - shift 2 - ;; - esac -done - -[[ -z ${clean:- } ]] || rm -rf "${scriptRoot}/build" -"${scriptRoot}/setup.py" build --build-number="$(git log -1 --pretty=format:%ct)" ${debugClient:-} diff --git a/examples/searchcommands_app/Build-App.ps1 b/examples/searchcommands_app/Build-App.ps1 deleted file mode 100644 index 96ce6f3ae..000000000 --- a/examples/searchcommands_app/Build-App.ps1 +++ /dev/null @@ -1,31 +0,0 @@ -[CmdletBinding()] -param( - [parameter(Mandatory=$false)] - [switch] - $Clean, - [parameter(Mandatory=$false)] - [switch] - $DebugBuild -) - -$buildNumber = git log -1 --pretty=format:%ct - -$debugClient = if ($DebugBuild) { - "--debug-client=`"C:\Program Files (x86)\JetBrains\PyCharm\debug-eggs\pycharm-debug.egg`"" -} -else { - "" -} - -if ($Clean) { - Get-Item -ErrorAction SilentlyContinue "$PSScriptRoot\build", "${env:SPLUNK_HOME}\etc\apps\chunked_searchcommands" | Remove-Item -ErrorAction Stop -Force -Recurse -} - -$ErrorActionPreference = "Continue" ;# Because PowerShell assumes a command has failed if there's any output to stderr even if the command's exit code is zero - -python "${PSScriptRoot}\setup.py" build --build-number="${buildNumber}" $debugClient - -if ($LASTEXITCODE -ne 0) { - "Exit code = $LASTEXITCODE" - return -} diff --git a/examples/searchcommands_app/Install-App b/examples/searchcommands_app/Install-App deleted file mode 100755 index b693205a2..000000000 --- a/examples/searchcommands_app/Install-App +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/bash-prologue" ${BASH_SOURCE[0]} 'help,clean,debug-client:' 'hcd:' $* || exit $? - -########### -# Arguments -########### - -eval set -- $args - -while [[ $1 != '--' ]] -do - case $1 in - -h|--help) - usage; # does not return - shift 1 - ;; - -c|--clean) - declare -r clean="clean" - shift 1 - ;; - -d|--debug-client) - [[ -f "$d" ]] || error 1 "Debug client '$2' does not exist." - declare -r debugClient="--debug-client '$2'" - shift 2 - ;; - esac -done - -# TODO: Answer this: We like "splunk restart -f" because it's fast, but what's the right thing to do for customers? -# TODO: Do the right thing when SPLUNK_HOME is undefined -# TODO: Parameterize version number - -declare -r appName="$(basename '${scriptRoot}')" -declare -r buildNumber=$(git log -1 --pretty=format:%ct) - -[[ -z ${clean:-} ]] || rm -rf "$scriptRoot/build" "${SPLUNK_HOME}/etc/apps/${appName}" -"${scriptRoot}/setup.py" build --build-number="$buildNumber" ${debugClient:-} -splunk start ;# Because the splunk daemon might not be running -splunk install app "${scriptRoot}\build\${appName}-1.0.0-${buildNumber}.tar.gz" -auth admin:changeme -update 1 -splunk restart -f ;# Because a restart is usually required after installing an application diff --git a/examples/searchcommands_app/Install-App.ps1 b/examples/searchcommands_app/Install-App.ps1 deleted file mode 100644 index cad80403e..000000000 --- a/examples/searchcommands_app/Install-App.ps1 +++ /dev/null @@ -1,58 +0,0 @@ -[CmdletBinding()] -param( - [parameter(Mandatory=$false)] - [switch] - $Clean, - [ValidateScript(ScriptBlock={Test-Path $_})] - [parameter(Mandatory=$false)] - [string] - $DebugClient -) - -# TODO: Answer this: We like "splunk restart -f" because it's fast, but what's the right thing to do for customers? -# TODO: Do the right thing when SPLUNK_HOME is undefined -# TODO: Parameterize version number - -$appName = Split-Path -Leaf $PSScriptRoot -$buildNumber = git log -1 --pretty=format:%ct - -$debugClient = if ($DebugClient -ne $null) { - "--debug-client=`"$DebugClient`"" -} -else { - "" -} - -if ($Clean) { - Get-Item -ErrorAction SilentlyContinue "$PSScriptRoot\build", "${env:SPLUNK_HOME}\etc\apps\${appName}" | Remove-Item -ErrorAction Stop -Force -Recurse -} - -$ErrorActionPreference = "Continue" ;# Because PowerShell assumes a command has failed if there's any output to stderr even if the command's exit code is zero - -python "${PSScriptRoot}\setup.py" build --build-number="${buildNumber}" $debugClient - -if ($LASTEXITCODE -ne 0) { - "Exit code = $LASTEXITCODE" - return -} - -splunk start ;# Because the splunk daemon might not be running - -if ($LASTEXITCODE -ne 0) { - "Exit code = $LASTEXITCODE" - return -} - -splunk install app "${PSScriptRoot}\build\${appName}-1.0.0-${buildNumber}.tar.gz" -auth admin:changeme -update 1 - -if ($LASTEXITCODE -ne 0) { - "Exit code = $LASTEXITCODE" - return -} - -splunk restart -f ;# Because a restart is usually required after installing an application - -if ($LASTEXITCODE -ne 0) { - "Exit code = $LASTEXITCODE" - return -} diff --git a/examples/searchcommands_app/README.md b/examples/searchcommands_app/README.md index ac6d1be09..075253134 100644 --- a/examples/searchcommands_app/README.md +++ b/examples/searchcommands_app/README.md @@ -7,7 +7,6 @@ This app provides several examples of custom search commands that illustrate eac :---------------- |:-----------|:------------------------------------------------------------------------------------------- countmatches | Streaming | Counts the number of non-overlapping matches to a regular expression in a set of fields. generatetext | Generating | Generates a specified number of events containing a specified text string. - pypygeneratetext | Generating | Runs generatetext with the string 'PyPy'. simulate | Generating | Generates a sequence of events drawn from a csv file using repeated random sampling with replacement. generatehello | Generating | Generates a specified number of events containing the text string 'hello'. sum | Reporting | Adds all of the numbers in a set of fields. @@ -19,12 +18,10 @@ The app is tested on Splunk 5 and 6. Here is its manifest: ├── bin │   ├── countmatches.py .......... CountMatchesCommand implementation │ ├── generatetext.py .......... GenerateTextCommand implementation -│ ├── pypygeneratetext.py ...... Runs generatetext.py with PyPy │ ├── simulate.py .............. SimulateCommand implementation │ └── sum.py ................... SumCommand implementation ├── lib -| └── splunklib -│ └── searchcommands ....... splunklib.searchcommands module +| └── splunklib ................ splunklib module ├── default │ ├── data │ │   └── ui @@ -35,41 +32,83 @@ The app is tested on Splunk 5 and 6. Here is its manifest: │ ├── logging.conf ............. Python logging[3] configuration in ConfigParser[4] format │ └── searchbnf.conf ........... Search assistant configuration [5] └── metadata - └── local.meta ............... Permits the search assistant to use searchbnf.conf[6] + └── default.meta ............. Permits the search assistant to use searchbnf.conf[6] ``` **References** -[1] [app.conf](http://docs.splunk.com/Documentation/Splunk/latest/Admin/Appconf app.conf) -[2] [commands.conf](http://docs.splunk.com/Documentation/Splunk/latest/Admin/Commandsconf) -[3] [Python Logging HOWTO](http://docs.python.org/2/howto/logging.html) -[4] [ConfigParser—Configuration file parser](http://docs.python.org/2/library/configparser.html) -[5] [searchbnf.conf](http://docs.splunk.com/Documentation/Splunk/latest/admin/Searchbnfconf) -[6] [Set permissions in the file system](http://docs.splunk.com/Documentation/Splunk/latest/AdvancedDev/SetPermissions#Set_permissions_in_the_filesystem) +[1] [app.conf](https://docs.splunk.com/Documentation/Splunk/latest/Admin/Appconf app.conf) +[2] [commands.conf](https://docs.splunk.com/Documentation/Splunk/latest/Admin/Commandsconf) +[3] [Python Logging HOWTO](https://docs.python.org/2/howto/logging.html) +[4] [ConfigParser—Configuration file parser](https://docs.python.org/2/library/configparser.html) +[5] [searchbnf.conf](https://docs.splunk.com/Documentation/Splunk/latest/admin/Searchbnfconf) +[6] [Set permissions in the file system](https://docs.splunk.com/Documentation/Splunk/latest/AdvancedDev/SetPermissions#Set_permissions_in_the_filesystem) ## Installation -+ Link the app to $SPLUNK_HOME/etc/apps/searchcommands_app by running this command: ++ Bring up Dockerized Splunk with the app installed from the root of this repository via: ``` - ./setup.py link --scp-version {1|2} + SPLUNK_VERSION=latest docker compose up -d ``` -+ Or build a tarball to install on any Splunk instance by running this command: ++ When the `splunk` service is healthy (`health: starting` -> `healthy`) login and run test searches within the app via http://localhost:8000/en-US/app/searchcommands_app/search - ``` - ./setup.py build --scp-version {1|2} - ``` +### Example searches - The tarball is build as build/searchcommands_app-1.5.0-private.tar.gz. - -+ Then (re)start Splunk so that the app is recognized. +#### countmatches +``` +| inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text +``` +Results: +text | word_count +:----|:---| +excellent review my friend loved it yours always guppyman @GGreeny62... http://t.co/fcvq7NDHxl | 14 +Tú novia te ama mucho | 5 +... | -## Dashboards and Searches +#### filter +``` +| generatetext text="Hello world! How the heck are you?" count=6 \ +| filter predicate="(int(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')" +``` +Results: +Event | +:-----| +2. Hello Splunk! How the heck are you? | +4. Hello Splunk! How the heck are you? | +6. Hello Splunk! How the heck are you? | -+ TODO: Add saved search(es) for each example. +#### generatetext +``` +| generatetext count=3 text="Hello there" +``` +Results: +Event | +:-----| +1. Hello there | +2. Hello there | +3. Hello there | -### Searches +#### simulate +``` +| simulate csv="/opt/splunk/etc/apps/searchcommands_app/data/population.csv" rate=10 interval=00:00:01 duration=00:00:02 seed=9 +``` +Results: +Event | +:-----| +text = Margarita (8) | +text = RT @Habibies: When you were born, you cried and the world rejoiced. Live your life so that when you die, the world will cry and you will re... | +text = @dudaribeiro_13 q engraçado em. | -+ TODO: Describe saved searches. +#### sum +``` +| inputlookup tweets +| countmatches fieldname=word_count pattern="\\w+" text +| sum total=word_counts word_count +``` +Results: +word_counts | +:-----| +4497.0 | ## License diff --git a/examples/searchcommands_app/Select-SearchCommandsApp b/examples/searchcommands_app/Select-SearchCommandsApp deleted file mode 100755 index 3089cef1d..000000000 --- a/examples/searchcommands_app/Select-SearchCommandsApp +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/bash-prologue" ${BASH_SOURCE[0]} 'help,clean,debug-client:' 'hcd:' $* || exit $? - -if [[ $1 == scpv1-1.3 ]]; then - rm -f "${SPLUNK_HOME}/etc/apps/searchcommands_app" - cd "${SPLUNK_HOME}/etc/apps" - ln -s ~/Workspace/splunk-sdks/splunk-sdk-python.master/examples/searchcommands_app -elif [[ $1 == scpv1-1.5 ]]; then - "${scriptRoot}/setup.py" link --scp-version 1 -elif [[ $1 == scpv2-1.5 ]]; then - "${scriptRoot}/setup.py" link --scp-version 2 -else - error 1 "Unrecognized argument: $1" -fi - -splunk restart -f diff --git a/examples/searchcommands_app/Test-Performance b/examples/searchcommands_app/Test-Performance deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/searchcommands_app/Test-Performance.xlsx b/examples/searchcommands_app/Test-Performance.xlsx deleted file mode 100644 index 1dd7cab72..000000000 Binary files a/examples/searchcommands_app/Test-Performance.xlsx and /dev/null differ diff --git a/examples/searchcommands_app/bash-prologue b/examples/searchcommands_app/bash-prologue deleted file mode 100644 index 12a06b4d8..000000000 --- a/examples/searchcommands_app/bash-prologue +++ /dev/null @@ -1,43 +0,0 @@ -########### -# Variables -########### - -declare -r scriptRoot="$(cd "$(dirname "$1")" && pwd)" -declare -r scriptName="$(basename "$1")" -declare -r scriptLongOptions="$2" -declare -r scriptOptions="$3" - -shift 3 - -########### -# Functions -########### - -function usage { - - man "${scriptName}" - exit 0 -} - -function error { - echo "${scriptName} error: $2" 1>&2 - exit $1 -} - -########### -# Constants -########### - -# useful for printing text to console... - -declare -r b="$(tput bold)" ; # bold -declare -r n="$(tput sgr0)" ; # normal -declare -r u="$(tput smul)" ; # underline -declare -r u_="$(tput rmul)" ; # underline off (neither $n nor $b defeat $u) - -########### -# Arguments -########### - -declare args=$(getopt --name "$scriptName" --options "$scriptOptions" --longoptions "$scriptLongOptions" -- $* || exit 1) -set -o errexit -o nounset diff --git a/examples/searchcommands_app/package/bin/filter.py b/examples/searchcommands_app/package/bin/filter.py index f85615c17..3a29ca9b2 100755 --- a/examples/searchcommands_app/package/bin/filter.py +++ b/examples/searchcommands_app/package/bin/filter.py @@ -49,7 +49,7 @@ class FilterCommand(EventingCommand): .. code-block:: | generatetext text="Hello world! How the heck are you?" count=6 - | filter predicate="(long(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')" + | filter predicate="(int(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')" """ predicate = Option(doc=''' diff --git a/examples/searchcommands_app/package/bin/pypygeneratetext.py b/examples/searchcommands_app/package/bin/pypygeneratetext.py deleted file mode 100755 index adf2c3b35..000000000 --- a/examples/searchcommands_app/package/bin/pypygeneratetext.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# -# Copyright 2011-2015 Splunk, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"): you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Requirements: -# 1. PyPy is on splunkd's path. -# Ensure this by performing these operating system-dependent tasks: -# -# CentOS -# ------ -# Create or update /etc/sysconfig/splunk with a line that looks like this: -# -# 1 PATH=$PATH:/opt/pypy/bin -# -# P1 [ ] TODO: Verify that the instructions for putting PyPy on Splunk's PATH on CentOS work -# -# OS X -# ---- -# Edit /Library/LaunchAgents/com.splunk.plist and ensure that it looks like this: -# -# 1 -# 2 -# 3 -# 4 -# 5 Label -# 6 com.splunk -# 7 ProgramArguments -# 8 -# 9 /Users/david-noble/Workspace/Splunk/bin/splunk -# 10 start -# 11 --no-prompt -# 12 --answer-yes -# 13 -# 14 RunAtLoad -# 15 -# 16 EnvironmentVariables -# 17 -# 18 PATH -# 19 /usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/local/bin -# 20 -# 21 -# 22 -# -# Note lines 16-20 which extend PATH to include /opt/local/bin, the directory that the pypy executable is typically -# placed. -# -# Windows -# ------- -# Ensure that pypy.exe is on the system-wide Path environment variable. - -from __future__ import absolute_import, division, print_function, unicode_literals -import app -import sys -from os import environ, path - -splunkhome = environ['SPLUNK_HOME'] -sys.path.append(path.join(splunkhome, 'etc', 'apps', 'searchcommands_app', 'lib')) -from splunklib.searchcommands import app_root, execute - -pypy_argv = ['pypy', path.join(app_root, 'bin', 'generatetext.py')] + sys.argv[1:] -pypy_environ = dict(environ) -pypy_environ.pop('PYTHONPATH', None) # On Windows Splunk is a 64-bit service, but pypy is a 32-bit program -pypy_environ.pop('DYLD_LIBRARY_PATH', None) # On *nix Splunk includes shared objects that are incompatible with pypy - -execute('pypy', pypy_argv, pypy_environ) diff --git a/examples/searchcommands_app/package/bin/simulate.py b/examples/searchcommands_app/package/bin/simulate.py index 31d68423f..db223c71b 100755 --- a/examples/searchcommands_app/package/bin/simulate.py +++ b/examples/searchcommands_app/package/bin/simulate.py @@ -47,9 +47,7 @@ class SimulateCommand(GeneratingCommand): ##Example .. code-block:: - | simulate csv=population.csv rate=50 interval=00:00:01 - duration=00:00:05 | countmatches fieldname=word_count - pattern="\\w+" text | stats mean(word_count) stdev(word_count) + | simulate csv="/opt/splunk/etc/apps/searchcommands_app/data/population.csv" rate=10 interval=00:00:01 duration=00:00:02 seed=1 This example generates events drawn from repeated random sampling of events from :code:`tweets.csv`. Events are drawn at an average rate of 50 per second for a duration of 5 seconds. Events are piped to the example @@ -85,28 +83,20 @@ class SimulateCommand(GeneratingCommand): **Description:** Value for initializing the random number generator ''') def generate(self): - - if not self.records: - if self.seed is not None: - random.seed(self.seed) - self.records = [record for record in csv.DictReader(self.csv_file)] - self.lambda_value = 1.0 / (self.rate / float(self.interval)) + if self.seed is not None: + random.seed(self.seed) + records = [record for record in csv.DictReader(self.csv_file)] + lambda_value = 1.0 / (self.rate / float(self.interval)) duration = self.duration - while duration > 0: - count = int(round(random.expovariate(self.lambda_value))) + count = int(round(random.expovariate(lambda_value))) start_time = time.clock() - for record in random.sample(self.records, count): + for record in random.sample(records, count): yield record interval = time.clock() - start_time if interval < self.interval: time.sleep(self.interval - interval) duration -= max(interval, self.interval) - def __init__(self): - super(SimulateCommand, self).__init__() - self.lambda_value = None - self.records = None - dispatch(SimulateCommand, sys.argv, sys.stdin, sys.stdout, __name__) diff --git a/examples/searchcommands_app/package/default/commands-scpv1.conf b/examples/searchcommands_app/package/default/commands-scpv1.conf deleted file mode 100644 index 8c3408a86..000000000 --- a/examples/searchcommands_app/package/default/commands-scpv1.conf +++ /dev/null @@ -1,62 +0,0 @@ -# [commands.conf]($SPLUNK_HOME/etc/system/README/commands.conf.spec) -# Configuration for Search Commands Protocol version 1 - -[countmatches] -filename = countmatches.py -enableheader = true -outputheader = true -requires_srinfo = true -stderr_dest = message -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true - -[filter] -filename = filter.py -enableheader = true -outputheader = true -requires_srinfo = true -stderr_dest = message -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true - -[generatetext] -filename = generatetext.py -enableheader = true -outputheader = true -requires_srinfo = true -stderr_dest = message -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true - -[pypygeneratetext] -filename = pypygeneratetext.py -enableheader = true -outputheader = true -requires_srinfo = true -stderr_dest = message -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true - -[simulate] -filename = simulate.py -enableheader = true -outputheader = true -stderr_dest = message -requires_srinfo = true -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true - -[sum] -filename = sum.py -enableheader = true -outputheader = true -requires_srinfo = true -stderr_dest = message -supports_getinfo = true -supports_rawargs = true -supports_multivalues = true diff --git a/examples/searchcommands_app/package/default/commands-scpv2.conf b/examples/searchcommands_app/package/default/commands.conf similarity index 75% rename from examples/searchcommands_app/package/default/commands-scpv2.conf rename to examples/searchcommands_app/package/default/commands.conf index 7aa773abf..4ef41c556 100644 --- a/examples/searchcommands_app/package/default/commands-scpv2.conf +++ b/examples/searchcommands_app/package/default/commands.conf @@ -4,23 +4,24 @@ [countmatches] filename = countmatches.py chunked = true +python.version = python3 [filter] filename = filter.py chunked = true +python.version = python3 [generatetext] filename = generatetext.py chunked = true - -[pypygeneratetext] -filename = pypygeneratetext.py -chunked = true +python.version = python3 [simulate] filename = simulate.py chunked = true +python.version = python3 [sum] filename = sum.py chunked = true +python.version = python3 diff --git a/examples/searchcommands_app/package/default/searchbnf.conf b/examples/searchcommands_app/package/default/searchbnf.conf index 059815803..8254f027d 100644 --- a/examples/searchcommands_app/package/default/searchbnf.conf +++ b/examples/searchcommands_app/package/default/searchbnf.conf @@ -35,7 +35,7 @@ comment1 = \ of the records produced by the generatetext command. example1 = \ | generatetext text="Hello world! How the heck are you?" count=6 \ - | filter predicate="(long(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')" + | filter predicate="(int(_serial) & 1) == 0" update="_raw = _raw.replace('world', 'Splunk')" category = events appears-in = 1.5 maintainer = dnoble @@ -58,23 +58,6 @@ maintainer = dnoble usage = public tags = searchcommands_app -[pypygeneratetext-command] -syntax = PYPYGENERATETEXT COUNT= TEXT= -alias = -shortdesc = Generates a sequence of occurrences of a text string on the streams pipeline under control of PyPy. -description = \ - This command generates COUNT occurrences of a TEXT string under control of PyPy. Each occurrence is prefixed \ - by its _SERIAL number and stored in the _RAW field of each record. This command assumes that PyPy is on Splunkd's \ - PATH. -comment1 = \ - This example generates 10 occurrences of the string "Hello world!". -example1 = | pypygeneratetext count=10 text="Hello world!" -category = external generating -appears-in = 1.5 -maintainer = dnoble -usage = public -tags = searchcommands_app - [simulate-command] syntax = SIMULATE CSV= RATE= INTERVAL= DURATION= \ [SEED=]? @@ -90,9 +73,7 @@ comment1 = \ countmatches command which adds a word_count field containing the number of words in the text field of each event. \ The mean and standard deviation of the word_count are then computed by the builtin stats command. example1 = \ - | simulate csv=population.csv rate=50 interval=00:00:01 duration=00:00:05 \ - | countmatches fieldname=word_count pattern="\\w+" text \ - | stats mean(word_count) stdev(word_count) + | simulate csv="/opt/splunk/etc/apps/searchcommands_app/data/population.csv" rate=10 interval=00:00:01 duration=00:00:02 seed=1 category = generating appears-in = 1.2 maintainer = dnoble diff --git a/examples/searchcommands_app/setup.py b/examples/searchcommands_app/setup.py deleted file mode 100755 index ba2d46a0d..000000000 --- a/examples/searchcommands_app/setup.py +++ /dev/null @@ -1,490 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -# -# Copyright © Splunk, Inc. All rights reserved. - -from __future__ import absolute_import, division, print_function -import os - - -if os.name == 'nt': - - def patch_os(): - import ctypes - - kernel32 = ctypes.windll.kernel32 - format_error = ctypes.FormatError - - create_hard_link = kernel32.CreateHardLinkW - create_symbolic_link = kernel32.CreateSymbolicLinkW - delete_file = kernel32.DeleteFileW - get_file_attributes = kernel32.GetFileAttributesW - remove_directory = kernel32.RemoveDirectoryW - - def islink(path): - attributes = get_file_attributes(path) - return attributes != -1 and (attributes & 0x400) != 0 # 0x400 == FILE_ATTRIBUTE_REPARSE_POINT - - os.path.islink = islink - - def link(source, link_name): - if create_hard_link(link_name, source, None) == 0: - raise OSError(format_error()) - - os.link = link - - def remove(path): - attributes = get_file_attributes(path) - if attributes == -1: - success = False - elif (attributes & 0x400) == 0: # file or directory, not symbolic link - success = delete_file(path) != 0 - elif (attributes & 0x010) == 0: # symbolic link to file - success = delete_file(path) != 0 - else: # symbolic link to directory - success = remove_directory(path) != 0 - if success: - return - raise OSError(format_error()) - - os.remove = remove - - def symlink(source, link_name): - if create_symbolic_link(link_name, source, 1 if os.path.isdir(source) else 0) == 0: - raise OSError(format_error()) - - os.symlink = symlink - - patch_os() - del locals()['patch_os'] # since this function has done its job - -import sys -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) - -try: - from collections import OrderedDict -except: - from splunklib.ordereddict import OrderedDict - -from splunklib import six -from splunklib.six.moves import getcwd -from glob import glob -from itertools import chain -from setuptools import setup, Command -from subprocess import CalledProcessError, check_call, STDOUT - -import pip -import shutil -import sys - -project_dir = os.path.dirname(os.path.abspath(__file__)) - -# region Helper functions - - -def install_packages(app_root, distribution): - requires = distribution.metadata.requires - - if not requires: - return - - target = os.path.join(app_root, 'lib', 'packages') - - if not os.path.isdir(target): - os.mkdir(target) - - pip.main(['install', '--ignore-installed', '--target', target] + requires) - return - - -def splunk(*args): - check_call(chain(('splunk', ), args), stderr=STDOUT, stdout=sys.stdout) - return - - -def splunk_restart(uri, auth): - splunk('restart', "-uri", uri, "-auth", auth) - -# endregion - -# region Command definitions - - -class AnalyzeCommand(Command): - """ - setup.py command to run code coverage of the test suite. - - """ - description = 'Create an HTML coverage report from running the full test suite.' - - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - try: - from coverage import coverage - except ImportError: - print('Could not import the coverage package. Please install it and try again.') - exit(1) - return - c = coverage(source=['splunklib']) - c.start() - # TODO: instantiate and call TestCommand - # run_test_suite() - c.stop() - c.html_report(directory='coverage_report') - - -class BuildCommand(Command): - """ - setup.py command to create the application package file. - - """ - description = 'Package the app for distribution.' - - user_options = [ - ('build-number=', None, - 'Build number (default: private)'), - ('debug-client=', None, - 'Copies the file at the specified location to package/bin/_pydebug.egg and bundles it and _pydebug.conf ' - 'with the app'), - ('force', 'f', - 'Forcibly build everything'), - ('scp-version=', None, - 'Specifies the protocol version for search commands (default: 2)')] - - def __init__(self, dist): - - Command.__init__(self, dist) - - package = self.distribution.metadata - - self.package_name = '-'.join((package.name, package.version)) - self.build_base = os.path.join(project_dir, 'build') - self.build_dir = os.path.join(self.build_base, package.name) - self.build_lib = self.build_dir - - self.build_number = 'private' - self.debug_client = None - self.force = None - self.scp_version = 1 - - return - - def initialize_options(self): - return - - def finalize_options(self): - - self.scp_version = int(self.scp_version) - - if not (self.scp_version == 1 or self.scp_version == 2): - raise SystemError('Expected an SCP version number of 1 or 2, not {}'.format(self.scp_version)) - - self.package_name = self.package_name + '-' + six.text_type(self.build_number) - return - - def run(self): - - if self.force and os.path.isdir(self.build_dir): - shutil.rmtree(self.build_dir) - - self.run_command('build_py') - self._copy_package_data() - self._copy_data_files() - - if self.debug_client is not None: - try: - shutil.copy(self.debug_client, os.path.join(self.build_dir, 'bin', '_pydebug.egg')) - debug_conf = os.path.join(project_dir, 'package', 'bin', '_pydebug.conf') - if os.path.exists(debug_conf): - shutil.copy(debug_conf, os.path.join(self.build_dir, 'bin', '_pydebug.conf')) - except IOError as error: - print('Could not copy {}: {}'.format(error.filename, error.strerror)) - - install_packages(self.build_dir, self.distribution) - - # Link to the selected commands.conf as determined by self.scp_version (TODO: make this an install step) - - commands_conf = os.path.join(self.build_dir, 'default', 'commands.conf') - source = os.path.join(self.build_dir, 'default', 'commands-scpv{}.conf'.format(self.scp_version)) - - if os.path.isfile(commands_conf) or os.path.islink(commands_conf): - os.remove(commands_conf) - elif os.path.exists(commands_conf): - message = 'Cannot create a link at "{}" because a file by that name already exists.'.format(commands_conf) - raise SystemError(message) - - shutil.copy(source, commands_conf) - self._make_archive() - return - - def _copy_data_files(self): - for directory, path_list in self.distribution.data_files: - target = os.path.join(self.build_dir, directory) - if not os.path.isdir(target): - os.makedirs(target) - for path in path_list: - for source in glob(path): - if os.path.isfile(source): - shutil.copy(source, target) - pass - pass - pass - return - - def _copy_package_data(self): - for directory, path_list in six.iteritems(self.distribution.package_data): - target = os.path.join(self.build_dir, directory) - if not os.path.isdir(target): - os.makedirs(target) - for path in path_list: - for source in glob(path): - if os.path.isfile(source): - shutil.copy(source, target) - pass - pass - pass - return - - def _make_archive(self): - import tarfile - - build_dir = os.path.basename(self.build_dir) - archive_name = self.package_name + '.tar' - current_dir = getcwd() - os.chdir(self.build_base) - - try: - # We must convert the archive_name and base_dir from unicode to utf-8 due to a bug in the version of tarfile - # that ships with Python 2.7.2, the version of Python used by the app team's build system as of this date: - # 12 Sep 2014. - tar = tarfile.open(str(archive_name), 'w|gz') - try: - tar.add(str(build_dir)) - finally: - tar.close() - gzipped_archive_name = archive_name + '.gz' - if os.path.exists(gzipped_archive_name): - os.remove(gzipped_archive_name) - os.rename(archive_name, gzipped_archive_name) - finally: - os.chdir(current_dir) - - return - - -class LinkCommand(Command): - """ - setup.py command to create a symbolic link to the app package at $SPLUNK_HOME/etc/apps. - - """ - description = 'Create a symbolic link to the app package at $SPLUNK_HOME/etc/apps.' - - user_options = [ - ('debug-client=', None, 'Copies the specified PyCharm debug client egg to package/_pydebug.egg'), - ('scp-version=', None, 'Specifies the protocol version for search commands (default: 2)'), - ('splunk-home=', None, 'Overrides the value of SPLUNK_HOME.')] - - def __init__(self, dist): - Command.__init__(self, dist) - - self.debug_client = None - self.scp_version = 2 - self.splunk_home = os.environ['SPLUNK_HOME'] - self.app_name = self.distribution.metadata.name - self.app_source = os.path.join(project_dir, 'package') - - return - - def initialize_options(self): - pass - - def finalize_options(self): - - self.scp_version = int(self.scp_version) - - if not (self.scp_version == 1 or self.scp_version == 2): - raise SystemError('Expected an SCP version number of 1 or 2, not {}'.format(self.scp_version)) - - return - - def run(self): - target = os.path.join(self.splunk_home, 'etc', 'apps', self.app_name) - - if os.path.islink(target): - os.remove(target) - elif os.path.exists(target): - message = 'Cannot create a link at "{}" because a file by that name already exists.'.format(target) - raise SystemError(message) - - packages = os.path.join(self.app_source, 'lib') - - if not os.path.isdir(packages): - os.mkdir(packages) - - splunklib = os.path.join(packages, 'splunklib') - source = os.path.normpath(os.path.join(project_dir, '..', '..', 'splunklib')) - - if os.path.islink(splunklib): - os.remove(splunklib) - - os.symlink(source, splunklib) - - self._link_debug_client() - install_packages(self.app_source, self.distribution) - - commands_conf = os.path.join(self.app_source, 'default', 'commands.conf') - source = os.path.join(self.app_source, 'default', 'commands-scpv{}.conf'.format(self.scp_version)) - - if os.path.islink(commands_conf): - os.remove(commands_conf) - elif os.path.exists(commands_conf): - message = 'Cannot create a link at "{}" because a file by that name already exists.'.format(commands_conf) - raise SystemError(message) - - os.symlink(source, commands_conf) - os.symlink(self.app_source, target) - - return - - def _link_debug_client(self): - - if not self.debug_client: - return - - pydebug_egg = os.path.join(self.app_source, 'bin', '_pydebug.egg') - - if os.path.exists(pydebug_egg): - os.remove(pydebug_egg) - - os.symlink(self.debug_client, pydebug_egg) - - -class TestCommand(Command): - """ - setup.py command to run the whole test suite. - - """ - description = 'Run full test suite.' - - user_options = [ - ('commands=', None, 'Comma-separated list of commands under test or *, if all commands are under test'), - ('build-number=', None, 'Build number for the test harness'), - ('auth=', None, 'Splunk login credentials'), - ('uri=', None, 'Splunk server URI'), - ('env=', None, 'Test running environment'), - ('pattern=', None, 'Pattern to match test files'), - ('skip-setup-teardown', None, 'Skips test setup/teardown on the Splunk server')] - - def __init__(self, dist): - Command.__init__(self, dist) - - self.test_harness_name = self.distribution.metadata.name + '-test-harness' - self.uri = 'https://localhost:8089' - self.auth = 'admin:changeme' - self.env = 'test' - self.pattern = 'test_*.py' - self.skip_setup_teardown = False - - return - - def initialize_options(self): - pass # option values must be initialized before this method is called (so why is this method provided?) - - def finalize_options(self): - pass - - def run(self): - import unittest - - if not self.skip_setup_teardown: - try: - splunk( - 'search', '| setup environment="{0}"'.format(self.env), '-app', self.test_harness_name, - '-uri', self.uri, '-auth', self.auth) - splunk_restart(self.uri, self.auth) - except CalledProcessError as e: - sys.exit(e.returncode) - - current_directory = os.path.abspath(getcwd()) - os.chdir(os.path.join(project_dir, 'tests')) - print('') - - try: - suite = unittest.defaultTestLoader.discover('.', pattern=self.pattern) - unittest.TextTestRunner(verbosity=2).run(suite) # 1 = show dots, >1 = show all - finally: - os.chdir(current_directory) - - if not self.skip_setup_teardown: - try: - splunk('search', '| teardown', '-app', self.test_harness_name, '-uri', self.uri, '-auth', self.auth) - except CalledProcessError as e: - sys.exit(e.returncode) - - return - -# endregion - -current_directory = getcwd() -os.chdir(project_dir) - -try: - setup( - description='Custom Search Command examples', - name=os.path.basename(project_dir), - version='1.6.17', - author='Splunk, Inc.', - author_email='devinfo@splunk.com', - url='http://github.com/splunk/splunk-sdk-python', - license='http://www.apache.org/licenses/LICENSE-2.0', - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Other Environment', - 'Intended Audience :: Information Technology', - 'License :: Other/Proprietary License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Topic :: System :: Logging', - 'Topic :: System :: Monitoring'], - packages=[ - 'lib.splunklib', 'lib.splunklib.searchcommands' - ], - package_dir={ - 'lib': os.path.join('package', 'lib'), - 'lib.splunklib': os.path.join('..', '..', 'splunklib'), - 'lib.splunklib.searchcommands': os.path.join('..', '..', 'splunklib', 'searchcommands') - }, - package_data={ - 'bin': [ - os.path.join('package', 'bin', 'app.py'), - os.path.join('package', 'bin', 'countmatches.py'), - os.path.join('package', 'bin', 'filter.py'), - os.path.join('package', 'bin', 'generatehello.py'), - os.path.join('package', 'bin', 'generatetext.py'), - os.path.join('package', 'bin', 'pypygeneratetext.py'), - os.path.join('package', 'bin', 'simulate.py'), - os.path.join('package', 'bin', 'sum.py') - ] - }, - data_files=[ - ('README', [os.path.join('package', 'README', '*.conf.spec')]), - ('default', [os.path.join('package', 'default', '*.conf')]), - ('lookups', [os.path.join('package', 'lookups', '*.csv.gz')]), - ('metadata', [os.path.join('package', 'metadata', 'default.meta')]) - ], - requires=[], - - cmdclass=OrderedDict(( - ('analyze', AnalyzeCommand), - ('build', BuildCommand), - ('link', LinkCommand), - ('test', TestCommand)))) -finally: - os.chdir(current_directory) diff --git a/examples/twitted/twitted/metadata/local.meta b/examples/twitted/twitted/metadata/local.meta deleted file mode 100644 index 489fe41a2..000000000 --- a/examples/twitted/twitted/metadata/local.meta +++ /dev/null @@ -1,129 +0,0 @@ -[app/ui] -owner = admin -version = 20110524 - -[app/launcher] -owner = admin -version = 20110524 - -[viewstates/flashtimeline%3Agog49lc6] -access = read : [ * ] -owner = nobody -version = 20110524 - -[savedsearches/Top%20Sources] -owner = admin -version = 20110524 - -[savedsearches/Top%20Words] -owner = admin -version = 20110524 - -[savedsearches/Statuses,%20verified] -owner = admin -version = 20110524 - -[savedsearches/Statuses,%20enriched] -owner = admin -version = 20110524 - -[savedsearches/Statuses] -owner = admin -version = 20110524 - -[savedsearches/Users,%20most%20followers] -owner = admin -version = 20110524 - -[savedsearches/Users,%20most%20tweets] -owner = admin -version = 20110524 - -[savedsearches/Users,%20verified,%20most%20tweets] -owner = admin -version = 20110524 - -[savedsearches/Users,%20verified,%20most%20followers] -owner = admin -version = 20110524 - -[savedsearches/Users,%20most%20seen%20tweets] -owner = admin -version = 20110524 - -[viewstates/flashtimeline%3Agopz0n46] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Statuses%2C%20most%20retweeted] -owner = admin -version = 4.3 - -[viewstates/flashtimeline%3Agot9p0bd] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Users%2C%20most%20deletes] -owner = admin -version = 4.3 - -[viewstates/flashtimeline%3Agoxlionw] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Statuses%2C%20real-time] -owner = admin -version = 4.3 - -[viewstates/flashtimeline%3Agp1rbo5g] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Top%20Words%2C%20version%202] -owner = admin -version = 4.3 - -[viewstates/flashtimeline%3Agp3htyye] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Most%20mentioned] -owner = admin -version = 4.3 - -[viewstates/flashtimeline%3Agp3hzuqr] -access = read : [ * ] -owner = nobody -version = 4.3 - -[savedsearches/Popular%20hashtags] -owner = admin -version = 4.3 - -[indexes/twitter] -owner = itay -version = 4.2.2 - -[viewstates/flashtimeline%3Agpsrhije] -access = read : [ * ] -owner = nobody -version = 4.2.2 - -[savedsearches/Top%20Tags] -owner = itay -version = 4.2.2 - -[] -access = read : [ * ], write : [ admin, power ] -export = none -version = 4.2.2 - -[inputs/tcp%3A%2F%2F9002] -owner = itay -version = 4.2.2 - diff --git a/setup.py b/setup.py index 903a1407e..284c50983 100755 --- a/setup.py +++ b/setup.py @@ -15,13 +15,9 @@ # under the License. from setuptools import setup, Command -from contextlib import closing -from subprocess import check_call, STDOUT import os import sys -import shutil -import tarfile import splunklib @@ -130,84 +126,6 @@ def run(self): run_test_suite_with_junit_output() -class DistCommand(Command): - """setup.py command to create .spl files for modular input and search - command examples""" - description = "Build modular input and search command example tarballs." - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - @staticmethod - def get_python_files(files): - """Utility function to get .py files from a list""" - python_files = [] - for file_name in files: - if file_name.endswith(".py"): - python_files.append(file_name) - - return python_files - - def run(self): - # Create random_numbers.spl and github_forks.spl - - app_names = ['random_numbers', 'github_forks'] - splunklib_arcname = "splunklib" - modinput_dir = os.path.join(splunklib_arcname, "modularinput") - - if not os.path.exists("build"): - os.makedirs("build") - - for app in app_names: - with closing(tarfile.open(os.path.join("build", app + ".spl"), "w")) as spl: - spl.add( - os.path.join("examples", app, app + ".py"), - arcname=os.path.join(app, "bin", app + ".py") - ) - - spl.add( - os.path.join("examples", app, "default", "app.conf"), - arcname=os.path.join(app, "default", "app.conf") - ) - spl.add( - os.path.join("examples", app, "README", "inputs.conf.spec"), - arcname=os.path.join(app, "README", "inputs.conf.spec") - ) - - splunklib_files = self.get_python_files(os.listdir(splunklib_arcname)) - for file_name in splunklib_files: - spl.add( - os.path.join(splunklib_arcname, file_name), - arcname=os.path.join(app, "bin", splunklib_arcname, file_name) - ) - - modinput_files = self.get_python_files(os.listdir(modinput_dir)) - for file_name in modinput_files: - spl.add( - os.path.join(modinput_dir, file_name), - arcname=os.path.join(app, "bin", modinput_dir, file_name) - ) - - spl.close() - - # Create searchcommands_app--private.tar.gz - # but only if we are on 2.7 or later - if sys.version_info >= (2,7): - setup_py = os.path.join('examples', 'searchcommands_app', 'setup.py') - - check_call(('python', setup_py, 'build', '--force'), stderr=STDOUT, stdout=sys.stdout) - tarball = 'searchcommands_app-{0}-private.tar.gz'.format(self.distribution.metadata.version) - source = os.path.join('examples', 'searchcommands_app', 'build', tarball) - target = os.path.join('build', tarball) - - shutil.copyfile(source, target) - - return - setup( author="Splunk, Inc.", @@ -215,8 +133,7 @@ def run(self): cmdclass={'coverage': CoverageCommand, 'test': TestCommand, - 'testjunit': JunitXmlTestCommand, - 'dist': DistCommand}, + 'testjunit': JunitXmlTestCommand}, description="The Splunk Software Development Kit for Python.", diff --git a/tests/searchcommands/test_searchcommands_app.py b/tests/searchcommands/test_searchcommands_app.py index 70bae6124..faf14abd8 100755 --- a/tests/searchcommands/test_searchcommands_app.py +++ b/tests/searchcommands/test_searchcommands_app.py @@ -199,26 +199,6 @@ def test_generatehello_as_unit(self): return - @skipUnless(pypy(), 'Skipping TestSearchCommandsApp.test_pypygeneratetext_as_unit because pypy is not on PATH.') - def test_pypygeneratetext_as_unit(self): - - expected, output, errors, exit_status = self._run_command('pypygeneratetext', action='getinfo', protocol=1) - self.assertEqual(0, exit_status, msg=six.text_type(errors)) - self.assertEqual('', errors, msg=six.text_type(errors)) - self._compare_csv_files_time_sensitive(expected, output) - - expected, output, errors, exit_status = self._run_command('pypygeneratetext', action='execute', protocol=1) - self.assertEqual(0, exit_status, msg=six.text_type(errors)) - self.assertEqual('', errors, msg=six.text_type(errors)) - self._compare_csv_files_time_insensitive(expected, output) - - expected, output, errors, exit_status = self._run_command('pypygeneratetext') - self.assertEqual(0, exit_status, msg=six.text_type(errors)) - self.assertEqual('', errors, msg=six.text_type(errors)) - self._compare_chunks(expected, output, time_sensitive=False) - - return - def test_sum_as_unit(self): expected, output, errors, exit_status = self._run_command('sum', action='getinfo', phase='reduce', protocol=1) diff --git a/tests/test_examples.py b/tests/test_examples.py index 3b63fc6da..b187dbbbd 100755 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -36,7 +36,6 @@ DIR_PATH = os.path.dirname(os.path.realpath(__file__)) EXAMPLES_PATH = os.path.join(DIR_PATH, '..', 'examples') -BUILD_PATH = os.path.join(DIR_PATH, '..', 'build') def check_multiline(testcase, first, second, message=None): """Assert that two multi-line strings are equal.""" @@ -96,9 +95,6 @@ def test_async(self): except: pass - def test_build_dir_exists(self): - self.assertTrue(os.path.exists(BUILD_PATH), 'Run setup.py build, then setup.py dist') - def test_binding1(self): result = run("binding1.py") self.assertEqual(result, 0)