From 785b8c4a30644006a938385a0ffc997707449215 Mon Sep 17 00:00:00 2001 From: Dmitry Gorodnichy Date: Thu, 16 May 2024 11:57:00 +0300 Subject: [PATCH 1/2] Add pagination during fetching articles --- bin/from_devto | 146 ++++++++++++++++++++++++++----------------------- 1 file changed, 79 insertions(+), 67 deletions(-) diff --git a/bin/from_devto b/bin/from_devto index f130818ae..d4f1d3f84 100755 --- a/bin/from_devto +++ b/bin/from_devto @@ -1,91 +1,100 @@ - #!/usr/bin/env ruby +#!/usr/bin/env ruby - class SaveArticleToMarkdown - require 'date' - require 'time' - require 'net/http' - require 'json' +class SaveArticleToMarkdown + require 'date' + require 'time' + require 'net/http' + require 'json' - DEV_TO_API_HOST = 'https://dev.to/api/articles/jetthoughts/'.freeze - JT_BLOG_HOST = 'https://jetthoughts.github.io/blog/'.freeze + DEV_TO_API_HOST = 'https://dev.to/api/articles/jetthoughts/'.freeze + JT_BLOG_HOST = 'https://jetthoughts.github.io/blog/'.freeze - def initialize(article_slug) - @article_slug = article_slug - end + def initialize(article_slug) + @article_slug = article_slug + end - def self.call(article_slug) - new(article_slug).call - end + def self.call(article_slug) + new(article_slug).call + end - def call - data = fetch_article_data - save_to_md_file(data) - update_canonical_url_on_dev_to(data) - fill_seo_attributes(data) - end + def call + data = fetch_article_data + save_to_md_file(data) + update_canonical_url_on_dev_to(data) + fill_seo_attributes(data) + end - def fetch_article_data - uri = URI(DEV_TO_API_HOST + @article_slug) - response = Net::HTTP.get(uri) - JSON.parse(response) - end + def fetch_article_data + uri = URI(DEV_TO_API_HOST + @article_slug) + response = Net::HTTP.get(uri) + JSON.parse(response) + end - def save_to_md_file(data) - # save to file related to this script - File.open(stored_post_path_for(data), 'w') { |file| file.write(data['body_markdown']) } - puts "File #{slug(data)}.md successfully created." - end + def save_to_md_file(data) + File.open(stored_post_path_for(data), 'w') { |file| file.write(data['body_markdown']) } + puts "File #{slug(data)}.md successfully created." + end - def stored_post_path_for(data) - File.expand_path(File.dirname(__FILE__)) + "/../content/blog/#{slug(data)}.md" - end + def stored_post_path_for(data) + __dir__ + "/../content/blog/#{slug(data)}.md" + end - def slug(data) - Time.parse(data["created_at"]).utc.strftime('%y%m%d') + '-' + data['slug'].split('-')[0..-2].join('-') - end + def slug(data) + Time.parse(data['created_at']).utc.strftime('%y%m%d') + '-' + data['slug'].split('-')[0..-2].join('-') + end - def update_canonical_url_on_dev_to(data) - uri = URI("https://dev.to/api/articles/#{data['id']}") + def update_canonical_url_on_dev_to(data) + uri = URI("https://dev.to/api/articles/#{data['id']}") - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = true + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true - request = Net::HTTP::Put.new(uri) - request['api-key'] = 'dAebJE2LywhZD6xBbAfLnfRK' - request['Content-Type'] = 'application/json' - pp JT_BLOG_HOST + slug(data) + request = Net::HTTP::Put.new(uri) + request['api-key'] = 'dAebJE2LywhZD6xBbAfLnfRK' + request['Content-Type'] = 'application/json' + pp JT_BLOG_HOST + slug(data) - # request.body = { article: { canonical_url: JT_BLOG_HOST + slug(data) } }.to_json - # http.request(request) - end + # request.body = { article: { canonical_url: JT_BLOG_HOST + slug(data) } }.to_json + # http.request(request) + end - def fill_seo_attributes(data) - markdown = File.read(stored_post_path_for(data)) - - File.open(stored_post_path_for(data), 'w') do |file| - file.write("+++\n") - file.write("title = #{data['title'].to_json}\n") - file.write("description = #{data['description'].to_json}\n") - file.write("created_at = \"#{data['created_at']}\"\n") - file.write("edited_at = \"#{data['edited_at']}\"\n") - file.write("sync_date = \"#{Time.now.utc.strftime('%Y-%m-%dT%H:%M:%SZ')}\"\n") - file.write("draft = false\n") - file.write("tags = #{data['tags']}\n") - file.write("+++\n") - file.write(markdown) - end + def fill_seo_attributes(data) + markdown = File.read(stored_post_path_for(data)) + + File.open(stored_post_path_for(data), 'w') do |file| + file.write("+++\n") + file.write("title = #{data['title'].to_json}\n") + file.write("description = #{data['description'].to_json}\n") + file.write("created_at = \"#{data['created_at']}\"\n") + file.write("edited_at = \"#{data['edited_at']}\"\n") + file.write("sync_date = \"#{Time.now.utc.strftime('%Y-%m-%dT%H:%M:%SZ')}\"\n") + file.write("draft = false\n") + file.write("tags = #{data['tags']}\n") + file.write("canonical_url = \"#{data['canonical_url']}\"\n") + file.write("slug = \"#{slug(data)}\"\n") + file.write("+++\n") + file.write(markdown) end end +end + +page = 1 +force = ARGV.include?('-f') ? true : false - uri = URI('https://dev.to/api/articles?username=jetthoughts') +def fetch_articles(page) + uri = URI("https://dev.to/api/articles?username=jetthoughts&page=#{page}") response = Net::HTTP.get(uri) - all_articles = JSON.parse(response) + articles = JSON.parse(response) - force = ARGV.include?('-f') ? true : false + articles.empty? ? nil : articles +end - pp "TODO: Need to uncomment canonical url update when blog will be live." +loop do + articles = fetch_articles(page) - all_articles.each do |article| + break if articles.nil? || articles.empty? + + articles.each do |article| created_at = Time.parse(article['created_at']) if article['created_at'] edited_at = Time.parse(article['edited_at']) if article['edited_at'] @@ -93,3 +102,6 @@ SaveArticleToMarkdown.call(article['slug']) end end + + page += 1 +end From 0a3083e5d1538491112e611afe3d8fc3fd70226e Mon Sep 17 00:00:00 2001 From: Dmitry Gorodnichy Date: Thu, 16 May 2024 11:59:08 +0300 Subject: [PATCH 2/2] Add notice --- bin/from_devto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/from_devto b/bin/from_devto index d4f1d3f84..6fcf5025d 100755 --- a/bin/from_devto +++ b/bin/from_devto @@ -105,3 +105,5 @@ loop do page += 1 end + +pp "TODO: Need to uncomment canonical url update when blog will be live."