Add pagination during fetching articles

bin/from_devto

@@ -1,95 +1,109 @@
-  #!/usr/bin/env ruby
+#!/usr/bin/env ruby
 
-  class SaveArticleToMarkdown
-    require 'date'
-    require 'time'
-    require 'net/http'
-    require 'json'
+class SaveArticleToMarkdown
+  require 'date'
+  require 'time'
+  require 'net/http'
+  require 'json'
 
-    DEV_TO_API_HOST = 'https://dev.to/api/articles/jetthoughts/'.freeze
-    JT_BLOG_HOST = 'https://jetthoughts.github.io/blog/'.freeze
+  DEV_TO_API_HOST = 'https://dev.to/api/articles/jetthoughts/'.freeze
+  JT_BLOG_HOST = 'https://jetthoughts.github.io/blog/'.freeze
 
-    def initialize(article_slug)
-      @article_slug = article_slug
-    end
+  def initialize(article_slug)
+    @article_slug = article_slug
+  end
 
-    def self.call(article_slug)
-      new(article_slug).call
-    end
+  def self.call(article_slug)
+    new(article_slug).call
+  end
 
-    def call
-      data = fetch_article_data
-      save_to_md_file(data)
-      update_canonical_url_on_dev_to(data)
-      fill_seo_attributes(data)
-    end
+  def call
+    data = fetch_article_data
+    save_to_md_file(data)
+    update_canonical_url_on_dev_to(data)
+    fill_seo_attributes(data)
+  end
 
-    def fetch_article_data
-      uri = URI(DEV_TO_API_HOST + @article_slug)
-      response = Net::HTTP.get(uri)
-      JSON.parse(response)
-    end
+  def fetch_article_data
+    uri = URI(DEV_TO_API_HOST + @article_slug)
+    response = Net::HTTP.get(uri)
+    JSON.parse(response)
+  end
 
-    def save_to_md_file(data)
-      # save to file related to this script
-      File.open(stored_post_path_for(data), 'w') { |file| file.write(data['body_markdown']) }
-      puts "File #{slug(data)}.md successfully created."
-    end
+  def save_to_md_file(data)
+    File.open(stored_post_path_for(data), 'w') { |file| file.write(data['body_markdown']) }
+    puts "File #{slug(data)}.md successfully created."
+  end
 
-    def stored_post_path_for(data)
-      File.expand_path(File.dirname(__FILE__)) + "/../content/blog/#{slug(data)}.md"
-    end
+  def stored_post_path_for(data)
+    __dir__ + "/../content/blog/#{slug(data)}.md"
+  end
 
-    def slug(data)
-      Time.parse(data["created_at"]).utc.strftime('%y%m%d') + '-' + data['slug'].split('-')[0..-2].join('-')
-    end
+  def slug(data)
+    Time.parse(data['created_at']).utc.strftime('%y%m%d') + '-' + data['slug'].split('-')[0..-2].join('-')
+  end
 
-    def update_canonical_url_on_dev_to(data)
-      uri = URI("https://dev.to/api/articles/#{data['id']}")
+  def update_canonical_url_on_dev_to(data)
+    uri = URI("https://dev.to/api/articles/#{data['id']}")
 
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
 
-      request = Net::HTTP::Put.new(uri)
-      request['api-key'] = 'dAebJE2LywhZD6xBbAfLnfRK'
-      request['Content-Type'] = 'application/json'
-      pp JT_BLOG_HOST + slug(data)
+    request = Net::HTTP::Put.new(uri)
+    request['api-key'] = 'dAebJE2LywhZD6xBbAfLnfRK'
+    request['Content-Type'] = 'application/json'
+    pp JT_BLOG_HOST + slug(data)
 
-      # request.body = { article: { canonical_url: JT_BLOG_HOST + slug(data) } }.to_json
-      # http.request(request)
-    end
+    # request.body = { article: { canonical_url: JT_BLOG_HOST + slug(data) } }.to_json
+    # http.request(request)
+  end
 
-    def fill_seo_attributes(data)
-      markdown = File.read(stored_post_path_for(data))
-
-      File.open(stored_post_path_for(data), 'w') do |file|
-        file.write("+++\n")
-        file.write("title = #{data['title'].to_json}\n")
-        file.write("description = #{data['description'].to_json}\n")
-        file.write("created_at = \"#{data['created_at']}\"\n")
-        file.write("edited_at = \"#{data['edited_at']}\"\n")
-        file.write("sync_date = \"#{Time.now.utc.strftime('%Y-%m-%dT%H:%M:%SZ')}\"\n")
-        file.write("draft = false\n")
-        file.write("tags = #{data['tags']}\n")
-        file.write("+++\n")
-        file.write(markdown)
-      end
+  def fill_seo_attributes(data)
+    markdown = File.read(stored_post_path_for(data))
+
+    File.open(stored_post_path_for(data), 'w') do |file|
+      file.write("+++\n")
+      file.write("title = #{data['title'].to_json}\n")
+      file.write("description = #{data['description'].to_json}\n")
+      file.write("created_at = \"#{data['created_at']}\"\n")
+      file.write("edited_at = \"#{data['edited_at']}\"\n")
+      file.write("sync_date = \"#{Time.now.utc.strftime('%Y-%m-%dT%H:%M:%SZ')}\"\n")
+      file.write("draft = false\n")
+      file.write("tags = #{data['tags']}\n")
+      file.write("canonical_url = \"#{data['canonical_url']}\"\n")
+      file.write("slug = \"#{slug(data)}\"\n")
+      file.write("+++\n")
+      file.write(markdown)
     end
   end
+end
+
+page = 1
+force = ARGV.include?('-f') ? true : false
 
-  uri = URI('https://dev.to/api/articles?username=jetthoughts')
+def fetch_articles(page)
+  uri = URI("https://dev.to/api/articles?username=jetthoughts&page=#{page}")
   response = Net::HTTP.get(uri)
-  all_articles = JSON.parse(response)
+  articles = JSON.parse(response)
+
+  articles.empty? ? nil : articles
+end
 
-  force = ARGV.include?('-f') ? true : false
+loop do
+  articles = fetch_articles(page)
 
-  pp "TODO: Need to uncomment canonical url update when blog will be live."
+  break if articles.nil? || articles.empty?
 
-  all_articles.each do |article|
+  articles.each do |article|
     created_at = Time.parse(article['created_at']) if article['created_at']
     edited_at = Time.parse(article['edited_at']) if article['edited_at']
 
     if (created_at && (Time.now - created_at) < 1800) || (edited_at && (Time.now - edited_at) < 1800) || force
       SaveArticleToMarkdown.call(article['slug'])
     end
   end
+
+  page += 1
+end
+
+pp "TODO: Need to uncomment canonical url update when blog will be live."