Merge pull request #167 from istresearch/hotfix-1.2.1

Hotfix 1.2.1

.travis.yml

@@ -16,10 +16,10 @@ env:
     run_opts: ""
   - docker: 1
     dockerfile_name: Dockerfile
-    docker_tag_suffix: 1.2
+    docker_tag_suffix: 1.2.1
   - docker: 1
     dockerfile_name: Dockerfile.py2alpine
-    docker_tag_suffix: 1.2-alpine
+    docker_tag_suffix: 1.2.1-alpine
 
 install: true
 

README.md

@@ -51,10 +51,10 @@ To set up a pre-canned Scrapy Cluster test environment, make sure you have the l
 
 ## Documentation
 
-Please check out the official [Scrapy Cluster 1.2 documentation](http://scrapy-cluster.readthedocs.org/en/latest/) for more information on how everything works!
+Please check out the official [Scrapy Cluster 1.2.1 documentation](http://scrapy-cluster.readthedocs.org/en/latest/) for more information on how everything works!
 
 ## Branches
 
-The `master` branch of this repository contains the latest stable release code for `Scrapy Cluster 1.2`.
+The `master` branch of this repository contains the latest stable release code for `Scrapy Cluster 1.2.1`.
 
 The `dev` branch contains bleeding edge code and is currently working towards [Scrapy Cluster 1.3](https://github.com/istresearch/scrapy-cluster/milestone/3). Please note that not everything may be documented, finished, tested, or finalized but we are happy to help guide those who are interested.

crawler/tests/online.py

@@ -35,10 +35,10 @@ class CustomSpider(LinkSpider):
 class TestLinkSpider(TestCase):
 
     example_feed = "{\"allowed_domains\":null,\"allow_regex\":null,\""\
-        "crawlid\":\"abc12345\",\"url\":\"istresearch.com\",\"expires\":0,\""\
+        "crawlid\":\"abc12345\",\"url\":\"http://dmoztools.net/\",\"expires\":0,\""\
         "ts\":1461549923.7956631184,\"priority\":1,\"deny_regex\":null,\""\
         "cookie\":null,\"attrs\":null,\"appid\":\"test\",\"spiderid\":\""\
-        "link\",\"useragent\":null,\"deny_extensions\":null,\"maxdepth\":0}"
+        "test-link\",\"useragent\":null,\"deny_extensions\":null,\"maxdepth\":0}"
 
     def setUp(self):
         self.settings = get_project_settings()
@@ -75,7 +75,7 @@ def test_crawler_process(self):
         d = runner.crawl(CustomSpider)
         d.addBoth(lambda _: reactor.stop())
         # add crawl to redis
-        key = "test-spider:istresearch.com:queue"
+        key = "test-spider:dmoztools.net:queue"
         self.redis_conn.zadd(key, self.example_feed, -99)
 
         # run the spider, give 20 seconds to see the url, crawl it,

docker-compose.yml

@@ -2,27 +2,27 @@ version: '2'
 
 services:
   kafka_monitor:
-    image: istresearch/scrapy-cluster:kafka-monitor-1.2
+    image: istresearch/scrapy-cluster:kafka-monitor-1.2.1
     depends_on:
       - kafka
       - redis
     restart: always
   redis_monitor:
-    image: istresearch/scrapy-cluster:redis-monitor-1.2
+    image: istresearch/scrapy-cluster:redis-monitor-1.2.1
     depends_on:
       - kafka
       - redis
       - zookeeper
     restart: always
   crawler:
-    image: istresearch/scrapy-cluster:crawler-1.2
+    image: istresearch/scrapy-cluster:crawler-1.2.1
     depends_on:
       - kafka
       - redis
       - zookeeper
     restart: always
   rest:
-    image: istresearch/scrapy-cluster:rest-1.2
+    image: istresearch/scrapy-cluster:rest-1.2.1
     depends_on:
       - kafka
       - redis

docs/conf.py

@@ -56,9 +56,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.2'
+version = '1.2.1'
 # The full version, including alpha/beta/rc tags.
-release = '1.2'
+release = '1.2.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/topics/advanced/docker.rst

@@ -77,7 +77,7 @@ It is recommended you use docker compose to orchestrate your cluster with all of
 
 ::
 
-    image: istresearch/scrapy-cluster:kafka-monitor-1.2
+    image: istresearch/scrapy-cluster:kafka-monitor-1.2.1
     build:
       context: .
       dockerfile: docker/kafka-monitor/Dockerfile

docs/topics/advanced/rediskeys.rst

@@ -69,6 +69,6 @@ If you run the integration tests, there may be temporary Redis keys created that
 
 - **cluster:test** - Used when testing the Kafka Monitor can act and set a key in Redis
 
-- **test-spider:istresearch.com:queue** - Used when testing the crawler installation can interact with Redis and Kafka
+- **test-spider:dmoztools.net:queue** - Used when testing the crawler installation can interact with Redis and Kafka
 
 - **stats:crawler:<hostname>:test-spider:<window>** - Automatically created and destoryed during crawler testing by the stats collection mechanism settings.

docs/topics/changelog.rst

@@ -5,6 +5,15 @@ Change Log
 
 This page serves to document any changes made between releases.
 
+Scrapy Cluster 1.2.1
+--------------------
+
+Date: 01/19/2018
+
+- Fixes unit test syntax for link spider
+
+- Fixes docker version upgrade on Travis for continuous integration tests
+
 Scrapy Cluster 1.2
 ------------------
 

docs/topics/introduction/quickstart.rst

@@ -431,15 +431,15 @@ Which ever setup you chose, every process within should stay running for the rem
 
 ::
 
-    python kafka_monitor.py feed '{"url": "http://istresearch.com", "appid":"testapp", "crawlid":"abc123"}'
+    python kafka_monitor.py feed '{"url": "http://dmoztools.net", "appid":"testapp", "crawlid":"abc123"}'
 
 You will see the following output on the command line for that successful request:
 
 ::
 
     2015-12-22 15:45:37,457 [kafka-monitor] INFO: Feeding JSON into demo.incoming
     {
-        "url": "http://istresearch.com",
+        "url": "http://dmoztools.net",
         "crawlid": "abc123",
         "appid": "testapp"
     }
@@ -460,7 +460,7 @@ Crawl Request:
 
 ::
 
-    python kafka_monitor.py feed '{"url": "http://dmoz.org", "appid":"testapp", "crawlid":"abc1234", "maxdepth":1}'
+    python kafka_monitor.py feed '{"url": "http://dmoztools.net", "appid":"testapp", "crawlid":"abc1234", "maxdepth":1}'
 
 Now send an ``info`` action request to see what is going on with the
 crawl:

docs/topics/kafka-monitor/quickstart.rst

@@ -33,7 +33,7 @@ JSON Object feeder into your desired Kafka Topic. This takes a valid JSON object
 
 ::
 
-    $ python kafka_monitor.py feed '{"url": "http://istresearch.com", "appid":"testapp", "crawlid":"ABC123"}'
+    $ python kafka_monitor.py feed '{"url": "http://dmoztools.net", "appid":"testapp", "crawlid":"ABC123"}'
 
 The command line feed is very slow and should not be used in production. Instead, you should write your own continuously running application to feed Kafka the desired API requests that you require.
 
@@ -89,10 +89,10 @@ Feed an item
 
 ::
 
-    $ python kafka_monitor.py feed '{"url": "http://istresearch.com", "appid":"testapp", "crawlid":"ABC123"}'
+    $ python kafka_monitor.py feed '{"url": "http://dmoztools.net", "appid":"testapp", "crawlid":"ABC123"}'
     2016-01-05 15:14:44,829 [kafka-monitor] INFO: Feeding JSON into demo.incoming
     {
-        "url": "http://istresearch.com",
+        "url": "http://dmoztools.net",
         "crawlid": "ABC123",
         "appid": "testapp"
     }
@@ -116,8 +116,8 @@ If you have a :ref:`Crawler <crawler>` running, you should see the html come thr
         "response_headers": {
             <headers omitted>
         },
-        "response_url": "http://istresearch.com",
-        "url": "http://istresearch.com",
+        "response_url": "http://dmoztools.net",
+        "url": "http://dmoztools.net",
         "status_code": 200,
         "status_msg": "OK",
         "appid": "testapp",

docs/topics/rest/api.rst

@@ -156,7 +156,7 @@ Feed a crawl request
 
 ::
 
-    $ curl scdev:5343/feed -H "Content-Type: application/json" -d '{"url":"istresearch.com", "appid":"madisonTest", "crawlid":"abc123"}'
+    $ curl scdev:5343/feed -H "Content-Type: application/json" -d '{"url":"http://dmoztools.net", "appid":"madisonTest", "crawlid":"abc123"}'
 
 Feed a Stats request
 

elk/docker-compose.elk.yml

@@ -5,7 +5,7 @@ version: '2'
 
 services:
   kafka_monitor:
-    image: istresearch/scrapy-cluster:kafka-monitor-1.2
+    image: istresearch/scrapy-cluster:kafka-monitor-1.2.1
     volumes:
       - logs:/usr/src/app/logs
     environment:
@@ -16,7 +16,7 @@ services:
       - redis
     restart: always
   redis_monitor:
-    image: istresearch/scrapy-cluster:redis-monitor-1.2
+    image: istresearch/scrapy-cluster:redis-monitor-1.2.1
     volumes:
       - logs:/usr/src/app/logs
     environment:
@@ -28,7 +28,7 @@ services:
       - zookeeper
     restart: always
   crawler:
-    image: istresearch/scrapy-cluster:crawler-1.2
+    image: istresearch/scrapy-cluster:crawler-1.2.1
     volumes:
       - logs:/usr/src/app/logs
     environment:
@@ -40,7 +40,7 @@ services:
       - zookeeper
     restart: always
   rest:
-    image: istresearch/scrapy-cluster:rest-1.2
+    image: istresearch/scrapy-cluster:rest-1.2.1
     volumes:
       - logs:/usr/src/app/logs
     depends_on:

travis/docker.sh

@@ -39,7 +39,7 @@ if [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ] && [
     sudo docker rmi istresearch/scrapy-cluster:rest-test
 
     # log into docker
-    sudo docker login -e="$DOCKER_EMAIL" -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+    sudo docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
 
     # push new containers
     sudo docker push istresearch/scrapy-cluster