Fix AWS SDK conflict with s3 plugin, down to v2

.travis.yml

@@ -7,7 +7,7 @@ matrix:
       gemfile: Gemfile
     - rvm: 2.1.10
       os: linux
-      gemfile: gemfiles/Gemfile.fluentd-0.12
+      gemfile: gemfiles/Gemfile.td-agent-2.3.5
 
 script: bundle exec rake test
 

Makefile

@@ -12,22 +12,24 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 
-.PHONY: run run-v0.12 test test-v.012 install $(wildcard test/test_*.rb) $(wildcard test/**/test_*.rb) benchmark benchmark-remote
+.PHONY: run run-td-agent-2.3.5 test test-td-agent-2.3.5 install $(wildcard test/test_*.rb) $(wildcard test/**/test_*.rb) benchmark benchmark-remote
 
 all:
 	bundle install
 
 run:
 	bundle exec fluentd -v
 
-run-v0.12:
-	BUNDLE_GEMFILE=./gemfiles/Gemfile.fluentd-0.12 bundle exec fluentd -v
+run-td-agent-2.3.5:
+	RBENV_VERSION=2.1.10 BUNDLE_GEMFILE=./gemfiles/Gemfile.td-agent-2.3.5 bundle update
+	RBENV_VERSION=2.1.10 BUNDLE_GEMFILE=./gemfiles/Gemfile.td-agent-2.3.5 bundle exec fluentd -v
 
 test:
 	bundle exec rake test
 
-test-v.012:
-	BUNDLE_GEMFILE=./gemfiles/Gemfile.fluentd-0.12 bundle exec rake test
+test-td-agent-2.3.5:
+	RBENV_VERSION=2.1.10 BUNDLE_GEMFILE=./gemfiles/Gemfile.td-agent-2.3.5 bundle update
+	RBENV_VERSION=2.1.10 BUNDLE_GEMFILE=./gemfiles/Gemfile.td-agent-2.3.5 bundle exec rake test
 
 install:
 	bundle exec rake install:local

README.md

@@ -379,7 +379,7 @@ To launch `fluentd` process with this plugin for development, follow the steps b
 
 To launch using Fluentd v0.12, use `BUNDLE_GEMFILE` environment variable:
 
-    BUNDLE_GEMFILE=$PWD/gemfiles/Gemfile.fluentd-0.12 bundle exec fluentd -c /path/to/fluent.conf
+    BUNDLE_GEMFILE=$PWD/gemfiles/Gemfile.td-agent-2.3.5 bundle exec fluentd -c /path/to/fluent.conf
 
 ## Contributing
 

fluent-plugin-kinesis.gemspec

@@ -31,8 +31,15 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.1'
 
   spec.add_dependency "fluentd", ">= 0.12.35", "< 2"
-  spec.add_dependency "aws-sdk-kinesis", "~> 1"
-  spec.add_dependency "aws-sdk-firehose", "~> 1"
+  spec.add_dependency "aws-sdk", ">= 2.9.9", "< 4"
+  # TODO: fluent-plugin-s3 depends on v2 only.
+  # https://github.com/fluent/fluent-plugin-s3/issues/208
+  #
+  # This plugin is sometimes used with s3 plugin.
+  # Unless s3 plugin is updated to be available with v3,
+  # this plugin should depend on v2 only.
+  # spec.add_dependency "aws-sdk-kinesis", "~> 1"
+  # spec.add_dependency "aws-sdk-firehose", "~> 1"
   spec.add_dependency "google-protobuf", "~> 3"
 
   spec.add_development_dependency "bundler", "~> 1.10"

gemfiles/Gemfile.fluentd-0.12 → gemfiles/Gemfile.td-agent-2.3.5

@@ -17,4 +17,9 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
 gemspec path: ".."
 
-gem "fluentd", ">= 0.12.35", "< 0.14"
+# Specify related gems for td-agent v2.3.5
+# https://github.com/treasure-data/omnibus-td-agent/blob/release-2.3.5/config/projects/td-agent2.rb#L23
+gem "fluentd", "0.12.35"
+# https://github.com/treasure-data/omnibus-td-agent/blob/release-2.3.5/plugin_gems.rb#L13-L15
+gem "fluent-plugin-s3", "0.8.2"
+gem "aws-sdk", "2.9.9"

lib/fluent/plugin/kinesis_helper/client.rb

@@ -76,13 +76,25 @@ def client
 
       private
 
+      def aws_sdk_v2?
+        @aws_sdk_v2 ||= Gem.loaded_specs['aws-sdk-core'].version < Gem::Version.create('3')
+      end
+
       def client_class
         case request_type
         when :streams, :streams_aggregated
-          require 'aws-sdk-kinesis'
+          if aws_sdk_v2?
+            require 'aws-sdk'
+          else
+            require 'aws-sdk-kinesis'
+          end
           Aws::Kinesis::Client
         when :firehose
-          require 'aws-sdk-firehose'
+          if aws_sdk_v2?
+            require 'aws-sdk'
+          else
+            require 'aws-sdk-firehose'
+          end
           Aws::Firehose::Client
         end
       end

test/helper.rb

@@ -17,6 +17,9 @@
 def fluentd_v0_12?
   @fluentd_v0_12 ||= Gem.loaded_specs['fluentd'].version < Gem::Version.create('0.14')
 end
+def aws_sdk_v2?
+  @aws_sdk_v2 ||= Gem.loaded_specs['aws-sdk-core'].version < Gem::Version.create('3')
+end
 def driver_run(d, records, time: nil)
   time ||= event_time("2011-01-02 13:14:15 UTC")
   if fluentd_v0_12?
@@ -33,6 +36,11 @@ def driver_run(d, records, time: nil)
   require 'fluent/test/log'
   require 'fluent/test/driver/output'
 end
+if aws_sdk_v2?
+  require 'aws-sdk'
+else
+  require 'aws-sdk-firehose'
+end
 require_relative './dummy_server'
 require 'test/unit'
 require 'mocha/test_unit'

test/kinesis_helper/test_api.rb

@@ -14,7 +14,6 @@
 
 require_relative '../helper'
 require 'fluent/plugin/kinesis_helper/api'
-require 'aws-sdk-firehose'
 
 class KinesisHelperAPITest < Test::Unit::TestCase
   class Mock

test/kinesis_helper/test_client.rb

@@ -14,8 +14,6 @@
 
 require_relative '../helper'
 require 'fluent/plugin/kinesis_helper/client'
-require 'aws-sdk-core'
-require 'aws-sdk-firehose'
 
 class KinesisHelperClientTest < Test::Unit::TestCase
   class Mock