WIP: Added document to explain measurements in RITA

.github/release.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# based on: https://github.com/skx/github-action-publish-binaries/blob/master/upload-script
+#
+# Upload binary artifacts when a new release is made.
+#
+
+set -e
+
+# Ensure that the GITHUB_TOKEN secret is included
+if [[ -z "$GITHUB_TOKEN" ]]; then
+  echo "Set the GITHUB_TOKEN env variable."
+  exit 1
+fi
+
+# Ensure that the file path is present
+if [[ -z "$1" ]]; then
+  echo "Missing file (pattern) to upload."
+  exit 1
+fi
+
+# Only upload to non-draft releases
+IS_DRAFT=$(jq --raw-output '.release.draft' $GITHUB_EVENT_PATH)
+if [ "$IS_DRAFT" = true ]; then
+  echo "This is a draft, so nothing to do!"
+  exit 0
+fi
+
+# Run the build-script
+make docker-build
+docker container create --name rita quay.io/activecm/rita:latest
+docker container cp rita:/rita ./rita
+
+# Prepare the headers
+AUTH_HEADER="Authorization: token ${GITHUB_TOKEN}"
+
+# Build the Upload URL from the various pieces
+RELEASE_ID=$(jq --raw-output '.release.id' $GITHUB_EVENT_PATH)
+
+# For each matching file
+for file in $*; do
+    echo "Processing file ${file}"
+
+    FILENAME=$(basename ${file})
+    UPLOAD_URL="https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}/assets?name=${FILENAME}"
+    echo "$UPLOAD_URL"
+
+    # Upload the file
+    curl \
+        -sSL \
+        -XPOST \
+        -H "${AUTH_HEADER}" \
+        --upload-file "${file}" \
+        --header "Content-Type:application/octet-stream" \
+        "${UPLOAD_URL}"
+done

.github/workflows/release.yml

@@ -0,0 +1,15 @@
+name: Upload files to new release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  upload:
+    name: Upload Artifacts
+    runs-on: ubuntu-16.04
+    steps:
+      - uses: actions/checkout@v1
+      - run: .github/release.sh rita install.sh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,32 @@
+name: Rita tests
+
+on:
+  pull_request:
+    # Run tests except if only markdown files are changed
+    paths:
+      - '**'
+      - '!*.md'
+
+jobs:
+  test:
+    name: static and unit tests
+    runs-on: ubuntu-16.04
+    steps:
+      - uses: actions/checkout@v1
+      - run: make docker-test
+
+  # integration:
+  #   name: integration tests
+  #   runs-on: ubuntu-16.04
+  #   steps:
+  #     - uses: actions/checkout@v1
+  #     - run: |
+  #         make docker-build-test
+  #         docker container create --name rita --entrypoint /bin/sleep quay.io/activecm/rita:test 5m
+  #         docker container start rita
+  #         docker container exec rita tar cf vendor.tar vendor
+  #         docker container cp rita:/go/src/github.com/activecm/rita/rita ./rita
+  #         docker container cp rita:/go/src/github.com/activecm/rita/vendor.tar ./vendor.tar
+  #         docker container stop rita
+  #         tar xf vendor.tar
+  #     - run: make integration-test

Contributing.md

@@ -23,51 +23,33 @@ There are several ways to contribute code to the RITA project.
 Before diving in, follow the [Manual Installation Instructions](docs/Manual%20Installation.md)
 
 * Work on bug fixes:
-  * Find an issue you would like to work on in the Github tracker
+  * Find an issue you would like to work on in the Github tracker, especially [unassigned issues marked "good first issue"](https://github.com/activecm/rita/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22+no%3Aassignee)
   * Leave a comment letting us know you would like to work on it
-* Add tests:
-  * All too often code developed to meet milestones only undergoes
-  empirical, human testing
-  * We would love to see unit tests throughout RITA
-  * There are a few sections of this project that currently have unit tests. [Here](https://github.com/activecm/rita/blob/master/analysis/beacon/analyzer_test.go) is a good example of an existing unit test.
-  * Also when writing tests it is advisable to work backwards, start with what
-  result you want to get and then work backwards through the code
-  * When you're ready to test code run `go test ./...` from the root directory
-  of the project
-  * Feel free to refactor code to increase our ability to test it
 * Add new features:
-  * If you would like to become involved in the development effort, please hop
-   on our [OFTC channel at #activecm](https://webchat.oftc.net/?channels=activecm)
-   and chat about what is currently being worked on
+  * If you would like to become involved in the development effort, open a new issue or continue a discussion on an existing issue
 
 ### Running Static Tests
-* Golint
-  * Install [golint](https://github.com/golang/lint)
-  * Run `golint ./... | grep -v '^vendor/'` from the root RITA directory
-  * Fix any errors and run golint again to verify
-* Gofmt
-  * Run `gofmt -l . | grep -v '^vendor/'` from the root RITA directory to identify files containing styling errors
-  * Run `gofmt -w .` to automatically resolve gofmt errors
-* Go vet
-  * Run `go tool vet $(find . -name '*.go' | grep -v '/vendor/')` from the root RITA directory
-  * Fix any errors and run golint again to verify
-* Go test
-  * Run `go test -v -race ./...` from the root RITA directory
-  * Ensure that all unit tests have passed
+* You must have a RITA [development environment](https://github.com/activecm/rita/blob/master/docs/Manual%20Installation.md#installing-golang) set up and [golangci-lint](https://github.com/golangci/golangci-lint#install) installed to run the tests.
+* Check the [Makefile](https://github.com/activecm/rita/blob/master/Makefile) for all options. Currently you can run `make test`, `make static-test`, and `make unit-test`. There is also `make integration-test` and docker variants that will require you install docker as well.
 
 ### Reviewing Automated Test Results
-Automated tests are run against each commit on Travis CI. Build results may be viewed [here](https://travis-ci.org/activecm/rita).
+Automated tests are run against each pull request. Build results may be viewed [here](https://github.com/activecm/rita/actions).
 
 ### Gittiquette Summary
-* In order to contribute to RITA, you must fork it
-  * Do not `go get` or `git clone` your forked repo
-  * Instead, `git remote add` it to your existing RITA repository
-* Split a branch off of master `git checkout -b [a-new-branch]`
-* Push your commits to your remote if you wish to develop in the public
-* When your work is finished, pull down the latest master branch, and rebase
-your feature branch off of it
-* Submit a pull request on Github
+* In order to contribute to RITA, you must [fork it](https://github.com/activecm/rita/fork).
+* Once you have a forked repo you will need to clone it to a very specific path which corresponds to _the original repo location_. This is due to the way packages are imported in Go programs.
+  * `git clone [your forked repo git url] $GOPATH/src/github.com/activecm/rita` (note the destination path must include github.com/activecm/rita and not your own repo)
+* Add `https://github.com/activecm/rita` as a new remote so you can pull new changes.
+  * `git remote add upstream https://github.com/activecm/rita`
+* Split a branch off of master .
+  * `git checkout -b [your new feature]`
+* When your work is finished, pull the latest changes from the upstream master and rebase your changes on it.
+  * `git checkout master; git pull -r upstream master`
+  * `git checkout [your new feature]; git rebase master`
+* Push your commits to your repo and submit a pull request on Github.
+
+Further info can be found in the [Gittiquette doc](docs/RITA%20Gittiquette.md) under the guidelines and contributors sections.
 
 ### Common Issues
 * Building Rita using `go install` or `go build` yields a RITA version of `UNDEFINED`
-  * Use `make` or `make install`
+  * Use `make` or `make install` instead

Makefile

@@ -14,7 +14,8 @@ SRC := $(shell find . -path ./vendor -prune -o -type f -name '*.go' -print)
 # https://www.cmcrossroads.com/article/makefile-optimization-eval-and-macro-caching
 cache = $(if $(cached-$1),,$(eval cached-$1 := 1)$(eval cache-$1 := $($1)))$(cache-$1)
 
-# The first recipe defined will be called when `make` is run without a target
+# force rita to be rebuilt even if it's up to date
+.PHONY: rita
 rita: vendor $(SRC)
 	go build ${LDFLAGS}
 
@@ -71,7 +72,6 @@ docker-build:
 .PHONY: docker-build-test
 docker-build-test:
 	docker build -t quay.io/activecm/rita:test -f test.Dockerfile .
-	docker run --rm quay.io/activecm/rita:test make
 
 # Runs all tests inside docker container
 .PHONY: docker-test

Readme.md

@@ -7,7 +7,6 @@ Brought to you by [Active Countermeasures](https://www.activecountermeasures.com
 [![Build Status](https://travis-ci.org/activecm/rita.svg?branch=master)](https://travis-ci.org/activecm/rita)
 
 ---
-### What is Here
 
 RITA is an open source framework for network traffic analysis.
 
@@ -16,42 +15,52 @@ The framework ingests [Bro/Zeek Logs](https://www.zeek.org/) in TSV format, and
  - **DNS Tunneling Detection** Search for signs of DNS based covert channels
  - **Blacklist Checking**: Query blacklists to search for suspicious domains and hosts
 
-### Automatic Installation
-**The automatic installer is officially supported on Ubuntu 16.04 LTS, Security Onion\*, and CentOS 7**
+## Install
 
-* Download the latest `install.sh` file from the [release page](https://github.com/activecm/rita/releases/latest)
-* Make the installer executable: `chmod +x ./install.sh`
-* Run the installer: `sudo ./install.sh`
+Please see our recommended [System Requirements](docs/System%20Requirements.md) document if you wish to use RITA in a production environment.
 
-\* Please see the [Security Onion RITA wiki page](https://github.com/Security-Onion-Solutions/security-onion/wiki/RITA) for further information pertaining to using RITA on Security Onion.
+### Automated Install
+
+RITA provides an install script that works on Ubuntu 18.04 LTS, Ubuntu 16.04 LTS, Security Onion, and CentOS 7.
+
+Download the latest `install.sh` file [here](https://github.com/activecm/rita/releases/latest) and make it executable: `chmod +x ./install.sh`
+
+Then choose one of the following install methods:
+
+* `sudo ./install.sh` will install RITA as well as supported versions of Bro/Zeek and MongoDB. This is suitable if you want to get started as quickly as possible or you don't already have Bro/Zeek or MongoDB.
+
+* `sudo ./install.sh --disable-bro --disable-mongo` will install RITA only, without Bro/Zeek or MongoDB. You may also use these flags individually.
+  * If you choose not to install Bro/Zeek you will need to [provide your own logs](#obtaining-data-generating-brozeek-logs).
+  * If you choose not to install MongoDB you will need to configure RITA to [use your existing MongoDB server](docs/Mongo%20Configuration.md).
+
+### Docker Install
+
+See [here](docs/Docker%20Usage.md).
 
 ### Manual Installation
-To install each component of RITA by hand, [check out the instructions in the docs](docs/Manual%20Installation.md).
+
+To install each component of RITA by manually see [here](docs/Manual%20Installation.md).
 
 ### Upgrading RITA
+
 See [this guide](docs/Upgrading.md) for upgrade instructions.
 
 ### Getting Started
 
-#### System Requirements
-* Operating System - The preferred platform is 64-bit Ubuntu 16.04 LTS. The system should be patched and up to date using apt-get.
-* Processor (when installed alongside Bro/Zeek) - Two cores plus an additional core for every 100 Mb of traffic being captured. (three cores minimum). This should be dedicated hardware, as resource congestion with other VMs can cause packets to be dropped or missed.
-* Memory - 16GB minimum. 64GB if monitoring 100Mb or more of network traffic. 128GB if monitoring 1Gb or more of network traffic.
-* Storage - 300GB minimum. 1TB or more is recommended to reduce log maintenance.
-* Network - In order to capture traffic with Bro/Zeek, you will need at least 2 network interface cards (NICs). One will be for management of the system and the other will be the dedicated capture port. Intel NICs perform well and are recommended.
-
 #### Configuration File
+
 RITA's config file is located at `/etc/rita/config.yaml` though you can specify a custom path on individual commands with the `-c` command line flag.
 
 :exclamation: **IMPORTANT** :exclamation:
-* The `Filtering: InternalSubnets` section *must* be configured or you will not see any results in certain modules (e.g. beacons, long connections). If your network uses the standard RFC1918 internal IP ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) you just need uncomment the default `InternalSubnets` section already in the config file. Otherwise, adjust this section to match your environment. RITA's main purpose is to find the signs of a compromised internal system talking to an external system and will automatically exclude internal to internal connections and external to external connections from parts of the analysis.
+* The `Filtering: InternalSubnets` section *must* be configured or you will not see any results in certain modules (e.g. beacons, long connections). If your network uses the standard RFC1918 internal IP ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) you don't need to do anything as the default `InternalSubnets` section already has these. Otherwise, adjust this section to match your environment. RITA's main purpose is to find the signs of a compromised internal system talking to an external system and will automatically exclude internal to internal connections and external to external connections from parts of the analysis.
 
 You may also wish to change the defaults for the following option:
 * `Filtering: AlwaysInclude` - Ranges listed here are exempt from the filtering applied by the `InternalSubnets` setting. The main use for this is to include internal DNS servers so that you can see the source of any DNS queries made.
 
 Note that any value listed in the `Filtering` section should be in CIDR format. So a single IP of `192.168.1.1` would be written as `192.168.1.1/32`.
 
-#### Obtaining Data (Generating Bro/Zeek Logs):
+#### Obtaining Data (Generating Bro/Zeek Logs)
+
   * **Option 1**: Generate PCAPs outside of Bro/Zeek
     * Generate PCAP files with a packet sniffer ([tcpdump](http://www.tcpdump.org/), [wireshark](https://www.wireshark.org/), etc.)
     * (Optional) Merge multiple PCAP files into one PCAP file
@@ -65,11 +74,12 @@ Note that any value listed in the `Filtering` section should be in CIDR format.
         * Provide the `--disable-bro` flag when running the installer if you intend to compile Bro/Zeek from source
 
 #### Importing and Analyzing Data With RITA
+
 After installing RITA, setting up the `InternalSubnets` section of the config file, and collecting some Bro/Zeek logs, you are ready to begin hunting.
 
 Filtering and whitelisting happens at import time. These optional settings can be found alongside `InternalSubnets` in the configuration file.
 
-RITA will process Bro/Zeek TSV logs in both plaintext and gzip compressed formats. Note, if you are using Security Onion or Bro's JSON log output you will need to [switch back to traditional TSV output](https://securityonion.readthedocs.io/en/latest/bro.html#tsv).
+RITA can process TSV, JSON, and [JSON streaming](https://github.com/corelight/json-streaming-logs) Bro/Zeek log file formats. These logs can be either plaintext or gzip compressed.
 
   * **Option 1**: Create a One-Off Dataset
       * `rita import path/to/your/bro_logs dataset_name` creates a dataset from a collection of Bro/Zeek logs in a directory
@@ -102,6 +112,7 @@ rita import --rolling --numchunks 48 /opt/bro/logs/current 48-hour-dataset
 ```
 
 #### Examining Data With RITA
+
   * Use the **show-X** commands
       * `show-databases`: Print the datasets currently stored
       * `show-beacons`: Print hosts which show signs of C2 software
@@ -120,11 +131,14 @@ rita import --rolling --numchunks 48 /opt/bro/logs/current 48-hour-dataset
 
 
 ### Getting help
+
 Please create an issue on GitHub if you have any questions or concerns.
 
 ### Contributing to RITA
+
 To contribute to RITA visit our [Contributing Guide](Contributing.md)
 
 ### License
+
 GNU GPL V3
 © Active Countermeasures ™

commands/commands.go

@@ -51,6 +51,13 @@ var (
 		Usage: "Tests which databases would be deleted. Does not actually delete any data, nor prompt for confirmation",
 	}
 
+	// deleteFlag indicates whether any matching, existing data should be deleted
+	// before importing the target data
+	deleteFlag = cli.BoolFlag{
+		Name:  "delete, D",
+		Usage: "Indicates that the existing dataset should be deleted before re-importing. If the dataset is a rolling dataset and --chunk is not specified, the latest chunk will be replaced.",
+	}
+
 	rollingFlag = cli.BoolFlag{
 		Name:  "rolling, R",
 		Usage: "Indicates rolling import, which builds on and removes data to maintain a fixed length of time",
@@ -83,6 +90,17 @@ var (
 		Usage: "Print a report instead of csv",
 	}
 
+	limitFlag = cli.IntFlag{
+		Name:  "limit, li",
+		Usage: "Limit the outputs of the result to `LIMIT` values",
+		Value: 1000,
+	}
+
+	noLimitFlag = cli.BoolFlag{
+		Name:  "no-limit, nl",
+		Usage: "No limit to the outputs of results",
+	}
+
 	blSortFlag = cli.StringFlag{
 		Name:  "sort, s",
 		Usage: "Sort by conn_count (# of connections), uconn_count (# of unique connections), total_bytes (# of bytes)",

commands/delete-database.go

@@ -110,7 +110,7 @@ func deleteDatabase(c *cli.Context) error {
 
 	// Iterate through databases to delete and delete them one by one
 	for _, database := range names {
-		dberr := deleteSingleDatabase(res, names, database, dryRun)
+		dberr := deleteSingleDatabase(res, database, dryRun)
 		if dberr != nil {
 			return cli.NewExitError(dberr.Error, -1)
 		}
@@ -124,10 +124,13 @@ func deleteDatabase(c *cli.Context) error {
 	return nil
 }
 
-func deleteSingleDatabase(res *resources.Resources, dbnames []string, db string, dryRun bool) error {
+func deleteSingleDatabase(res *resources.Resources, db string, dryRun bool) error {
 	// check if database exists
-	dbExists := util.StringInSlice(db, dbnames)
-
+	collNames, err := res.DB.Session.DB(db).CollectionNames()
+	if err != nil {
+		return err
+	}
+	dbExists := len(collNames) != 0
 	// check if metadatabase record for database exists
 	mDBExists := util.StringInSlice(db, res.MetaDB.GetDatabases())