Week 2

Gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gem 'ruby-prof'
+gem 'rspec-benchmark'
+gem 'ruby-progressbar'
+gem 'stackprof'
+gem 'pry'
+gem 'minitest'
+gem 'memory_profiler'

Makefile

@@ -0,0 +1,5 @@
+all_reports:
+	ruby ruby-prof-allocations.rb
+	ruby ruby-prof-memory.rb
+	ruby memory-profiler.rb
+	ruby benchmark.rb

benchmark.rb

@@ -0,0 +1,4 @@
+require_relative 'task-2'
+require 'benchmark'
+
+puts Benchmark.realtime { work("spec/fixtures/files/data_40000.txt", disable_gc: true) }

memory-profiler.rb

@@ -0,0 +1,8 @@
+require_relative 'task-2'
+require 'benchmark'
+require 'memory_profiler'
+
+report = MemoryProfiler.report do
+  work('spec/fixtures/files/data_60000.txt', true)
+end
+report.pretty_print(scale_bytes: true)

ruby-prof-allocations.rb

@@ -0,0 +1,20 @@
+require 'ruby-prof'
+require_relative 'task-2'
+
+RubyProf.measure_mode = RubyProf::ALLOCATIONS
+
+result = RubyProf::Profile.profile do
+  work("spec/fixtures/files/data_40000.txt", true)
+end
+
+printer = RubyProf::FlatPrinter.new(result)
+printer.print(File.open('ruby_prof_reports/flat_allocations.txt', 'w+'))
+
+printer = RubyProf::GraphHtmlPrinter.new(result)
+printer.print(File.open('ruby_prof_reports/graph_allocations.html', 'w+'))
+
+printer = RubyProf::CallStackPrinter.new(result)
+printer.print(File.open('ruby_prof_reports/callstack_allocations.html', 'w+'))
+
+printer = RubyProf::CallTreePrinter.new(result)
+printer.print(path: 'ruby_prof_reports', profile: 'profile')

ruby-prof-memory.rb

@@ -0,0 +1,20 @@
+require 'ruby-prof'
+require_relative 'task-2'
+
+RubyProf.measure_mode = RubyProf::MEMORY
+
+result = RubyProf::Profile.profile do
+  work("spec/fixtures/files/data_40000.txt", true)
+end
+
+printer = RubyProf::FlatPrinter.new(result)
+printer.print(File.open('ruby_prof_reports/flat_memory.txt', 'w+'))
+
+printer = RubyProf::GraphHtmlPrinter.new(result)
+printer.print(File.open('ruby_prof_reports/graph_memory.html', 'w+'))
+
+printer = RubyProf::CallStackPrinter.new(result)
+printer.print(File.open('ruby_prof_reports/callstack_memory.html', 'w+'))
+
+printer = RubyProf::CallTreePrinter.new(result)
+printer.print(path: 'ruby_prof_reports', profile: 'profile')

spec/fixtures/files/data.txt

@@ -0,0 +1,18 @@
+user,0,Leida,Cira,0
+session,0,0,Safari 29,87,2016-10-23
+session,0,1,Firefox 12,118,2017-02-27
+session,0,2,Internet Explorer 28,31,2017-03-28
+session,0,3,Internet Explorer 28,109,2016-09-15
+session,0,4,Safari 39,104,2017-09-27
+session,0,5,Internet Explorer 35,6,2016-09-01
+user,1,Palmer,Katrina,65
+session,1,0,Safari 17,12,2016-10-21
+session,1,1,Firefox 32,3,2016-12-20
+session,1,2,Chrome 6,59,2016-11-11
+session,1,3,Internet Explorer 10,28,2017-04-29
+session,1,4,Chrome 13,116,2016-12-28
+user,2,Gregory,Santos,86
+session,2,0,Chrome 35,6,2018-09-21
+session,2,1,Safari 49,85,2017-05-22
+session,2,2,Firefox 47,17,2018-02-02
+session,2,3,Chrome 20,84,2016-11-25

spec/rails_helper.rb

@@ -0,0 +1,5 @@
+require 'rspec-benchmark'
+
+RSpec.configure do |config|
+  config.include RSpec::Benchmark::Matchers
+end

spec/task-2_spec.rb

@@ -0,0 +1,29 @@
+require_relative './rails_helper'
+require_relative '../task-2'
+
+describe 'task-1' do
+  let(:result) { './spec/fixtures/files/result.json' }
+
+  describe 'Result' do
+    let(:file) { './spec/fixtures/files/data.txt' }
+
+    let(:expected_result) do
+      # '{"totalUsers":3,"uniqueBrowsersCount":14,"totalSessions":15,"allBrowsers":"CHROME 13,CHROME 20,CHROME 35,CHROME 6,FIREFOX 12,FIREFOX 32,FIREFOX 47,INTERNET EXPLORER 10,INTERNET EXPLORER 28,INTERNET EXPLORER 35,SAFARI 17,SAFARI 29,SAFARI 39,SAFARI 49","usersStats":{"Leida Cira":{"sessionsCount":6,"totalTime":"455 min.","longestSession":"118 min.","browsers":"FIREFOX 12, INTERNET EXPLORER 28, INTERNET EXPLORER 28, INTERNET EXPLORER 35, SAFARI 29, SAFARI 39","usedIE":true,"alwaysUsedChrome":false,"dates":["2017-09-27","2017-03-28","2017-02-27","2016-10-23","2016-09-15","2016-09-01"]},"Palmer Katrina":{"sessionsCount":5,"totalTime":"218 min.","longestSession":"116 min.","browsers":"CHROME 13, CHROME 6, FIREFOX 32, INTERNET EXPLORER 10, SAFARI 17","usedIE":true,"alwaysUsedChrome":false,"dates":["2017-04-29","2016-12-28","2016-12-20","2016-11-11","2016-10-21"]},"Gregory Santos":{"sessionsCount":4,"totalTime":"192 min.","longestSession":"85 min.","browsers":"CHROME 20, CHROME 35, FIREFOX 47, SAFARI 49","usedIE":false,"alwaysUsedChrome":false,"dates":["2018-09-21","2018-02-02","2017-05-22","2016-11-25"]}}}' + "\n"
+      '{"usersStats":{"Leida Cira":{"sessionsCount":6,"totalTime":"455 min.","longestSession":"118 min.","browsers":"FIREFOX 12, INTERNET EXPLORER 28, INTERNET EXPLORER 28, INTERNET EXPLORER 35, SAFARI 29, SAFARI 39","usedIE":true,"alwaysUsedChrome":false,"dates":["2017-09-27","2017-03-28","2017-02-27","2016-10-23","2016-09-15","2016-09-01"]},"Palmer Katrina":{"sessionsCount":5,"totalTime":"218 min.","longestSession":"116 min.","browsers":"CHROME 13, CHROME 6, FIREFOX 32, INTERNET EXPLORER 10, SAFARI 17","usedIE":true,"alwaysUsedChrome":false,"dates":["2017-04-29","2016-12-28","2016-12-20","2016-11-11","2016-10-21"]},"Gregory Santos":{"sessionsCount":4,"totalTime":"192 min.","longestSession":"85 min.","browsers":"CHROME 20, CHROME 35, FIREFOX 47, SAFARI 49","usedIE":false,"alwaysUsedChrome":false,"dates":["2018-09-21","2018-02-02","2017-05-22","2016-11-25"]}},"totalUsers":3,"uniqueBrowsersCount":14,"totalSessions":15,"allBrowsers":"CHROME 13,CHROME 20,CHROME 35,CHROME 6,FIREFOX 12,FIREFOX 32,FIREFOX 47,INTERNET EXPLORER 10,INTERNET EXPLORER 28,INTERNET EXPLORER 35,SAFARI 17,SAFARI 29,SAFARI 39,SAFARI 49"}'
+    end
+
+    it 'returns equal' do
+      work(file)
+      expect(File.read(result)).to eq(expected_result)
+    end
+  end
+
+  describe 'Memory usage' do
+    let(:data_file_path)  { "./data_large.txt" }
+
+    it 'performs success with data_large.txt' do
+      work(data_file_path)
+      expect((`ps -o rss= -p #{Process.pid}`.to_i / 1024)).to be < 38
+    end
+  end
+end

stackprof.rb

@@ -0,0 +1,6 @@
+require_relative 'task-2'
+require 'stackprof'
+
+StackProf.run(mode: :object, out: 'ruby_prof_reports/stackprof.dump', row: true) do
+  work('./spec/fixtures/files/data_20000.txt', true)
+end

task-2.rb

@@ -1,177 +1,92 @@
-# Deoptimized version of homework task
+# frozen_string_literal: true
 
 require 'json'
-require 'pry'
-require 'date'
-require 'minitest/autorun'
+require 'set'
 
-class User
-  attr_reader :attributes, :sessions
-
-  def initialize(attributes:, sessions:)
-    @attributes = attributes
-    @sessions = sessions
-  end
+def parse_line(line)
+  type, id, *params = line.split(',')
+  [type, id, *params]
 end
 
-def parse_user(user)
-  fields = user.split(',')
-  parsed_result = {
-    'id' => fields[1],
-    'first_name' => fields[2],
-    'last_name' => fields[3],
-    'age' => fields[4],
+def initialize_user(params)
+  {
+    user_key: "#{params[0]} #{params[1]}",
+    sessionsCount: 0,
+    totalTime: 0,
+    longestSession: 0,
+    usedIE: false,
+    alwaysUsedChrome: true,
+    dates: [],
+    browsers: []
   }
 end
 
-def parse_session(session)
-  fields = session.split(',')
-  parsed_result = {
-    'user_id' => fields[1],
-    'session_id' => fields[2],
-    'browser' => fields[3],
-    'time' => fields[4],
-    'date' => fields[5],
-  }
+def update_user_with_session(user, browser, time, date)
+  user[:browsers] << browser
+  user[:usedIE] ||= browser.include?('INTERNET EXPLORER')
+  user[:alwaysUsedChrome] &&= browser.include?('CHROME')
+  user[:totalTime] += time.to_i
+  user[:sessionsCount] += 1
+  user[:longestSession] = [user[:longestSession], time.to_i].max
+  user[:dates] << date
 end
 
-def collect_stats_from_users(report, users_objects, &block)
-  users_objects.each do |user|
-    user_key = "#{user.attributes['first_name']}" + ' ' + "#{user.attributes['last_name']}"
-    report['usersStats'][user_key] ||= {}
-    report['usersStats'][user_key] = report['usersStats'][user_key].merge(block.call(user))
-  end
+def build_user_stat(user)
+  {
+    sessionsCount: user[:sessionsCount],
+    totalTime: "#{user[:totalTime]} min.",
+    longestSession: "#{user[:longestSession]} min.",
+    browsers: user[:browsers].sort.join(', '),
+    usedIE: user[:usedIE],
+    alwaysUsedChrome: user[:alwaysUsedChrome],
+    dates: user[:dates].sort.reverse
+  }.to_json
 end
 
-def work
-  file_lines = File.read('data.txt').split("\n")
-
-  users = []
-  sessions = []
-
-  file_lines.each do |line|
-    cols = line.split(',')
-    users = users + [parse_user(line)] if cols[0] == 'user'
-    sessions = sessions + [parse_session(line)] if cols[0] == 'session'
+def work(file_path, disable_gc = false)
+  GC.disable if disable_gc
+
+  unique_browsers = Set.new
+  total_users = 0
+  total_sessions = 0
+  current_user = nil
+  is_first_user = true
+
+  File.open('spec/fixtures/files/result.json', 'w') do |result|
+    result.write('{"usersStats":{')
+
+    File.foreach(file_path, chomp: true) do |line|
+      type, _id, *params = parse_line(line)
+
+      if type == 'user'
+        if current_user
+          result.write(',') unless is_first_user
+          result.write("\"#{current_user[:user_key]}\":#{build_user_stat(current_user)}")
+          is_first_user = false
+        end
+
+        current_user = initialize_user(params)
+        total_users += 1
+      elsif type == 'session'
+        browser = params[1].upcase
+        update_user_with_session(current_user, browser, params[2], params[3])
+        unique_browsers.add(browser)
+        total_sessions += 1
+      end
+    end
+
+    if current_user
+      result.write(',') unless is_first_user
+      result.write("\"#{current_user[:user_key]}\":#{build_user_stat(current_user)}")
+    end
+
+    result.write('},')
+    result.write("\"totalUsers\":#{total_users},")
+    result.write("\"uniqueBrowsersCount\":#{unique_browsers.count},")
+    result.write("\"totalSessions\":#{total_sessions},")
+    result.write("\"allBrowsers\":\"#{unique_browsers.sort.join(',')}\"")
+    result.write('}')
   end
 
-  # Отчёт в json
-  #   - Сколько всего юзеров +
-  #   - Сколько всего уникальных браузеров +
-  #   - Сколько всего сессий +
-  #   - Перечислить уникальные браузеры в алфавитном порядке через запятую и капсом +
-  #
-  #   - По каждому пользователю
-  #     - сколько всего сессий +
-  #     - сколько всего времени +
-  #     - самая длинная сессия +
-  #     - браузеры через запятую +
-  #     - Хоть раз использовал IE? +
-  #     - Всегда использовал только Хром? +
-  #     - даты сессий в порядке убывания через запятую +
-
-  report = {}
-
-  report[:totalUsers] = users.count
-
-  # Подсчёт количества уникальных браузеров
-  uniqueBrowsers = []
-  sessions.each do |session|
-    browser = session['browser']
-    uniqueBrowsers += [browser] if uniqueBrowsers.all? { |b| b != browser }
-  end
-
-  report['uniqueBrowsersCount'] = uniqueBrowsers.count
-
-  report['totalSessions'] = sessions.count
-
-  report['allBrowsers'] =
-    sessions
-      .map { |s| s['browser'] }
-      .map { |b| b.upcase }
-      .sort
-      .uniq
-      .join(',')
-
-  # Статистика по пользователям
-  users_objects = []
-
-  users.each do |user|
-    attributes = user
-    user_sessions = sessions.select { |session| session['user_id'] == user['id'] }
-    user_object = User.new(attributes: attributes, sessions: user_sessions)
-    users_objects = users_objects + [user_object]
-  end
-
-  report['usersStats'] = {}
-
-  # Собираем количество сессий по пользователям
-  collect_stats_from_users(report, users_objects) do |user|
-    { 'sessionsCount' => user.sessions.count }
-  end
-
-  # Собираем количество времени по пользователям
-  collect_stats_from_users(report, users_objects) do |user|
-    { 'totalTime' => user.sessions.map {|s| s['time']}.map {|t| t.to_i}.sum.to_s + ' min.' }
-  end
-
-  # Выбираем самую длинную сессию пользователя
-  collect_stats_from_users(report, users_objects) do |user|
-    { 'longestSession' => user.sessions.map {|s| s['time']}.map {|t| t.to_i}.max.to_s + ' min.' }
-  end
-
-  # Браузеры пользователя через запятую
-  collect_stats_from_users(report, users_objects) do |user|
-    { 'browsers' => user.sessions.map {|s| s['browser']}.map {|b| b.upcase}.sort.join(', ') }
-  end
-
-  # Хоть раз использовал IE?
-  collect_stats_from_users(report, users_objects) do |user|
-    { 'usedIE' => user.sessions.map{|s| s['browser']}.any? { |b| b.upcase =~ /INTERNET EXPLORER/ } }
-  end
-
-  # Всегда использовал только Chrome?
-  collect_stats_from_users(report, users_objects) do |user|
-    { 'alwaysUsedChrome' => user.sessions.map{|s| s['browser']}.all? { |b| b.upcase =~ /CHROME/ } }
-  end
-
-  # Даты сессий через запятую в обратном порядке в формате iso8601
-  collect_stats_from_users(report, users_objects) do |user|
-    { 'dates' => user.sessions.map{|s| s['date']}.map {|d| Date.parse(d)}.sort.reverse.map { |d| d.iso8601 } }
-  end
-
-  File.write('result.json', "#{report.to_json}\n")
   puts "MEMORY USAGE: %d MB" % (`ps -o rss= -p #{Process.pid}`.to_i / 1024)
 end
-
-class TestMe < Minitest::Test
-  def setup
-    File.write('result.json', '')
-    File.write('data.txt',
-'user,0,Leida,Cira,0
-session,0,0,Safari 29,87,2016-10-23
-session,0,1,Firefox 12,118,2017-02-27
-session,0,2,Internet Explorer 28,31,2017-03-28
-session,0,3,Internet Explorer 28,109,2016-09-15
-session,0,4,Safari 39,104,2017-09-27
-session,0,5,Internet Explorer 35,6,2016-09-01
-user,1,Palmer,Katrina,65
-session,1,0,Safari 17,12,2016-10-21
-session,1,1,Firefox 32,3,2016-12-20
-session,1,2,Chrome 6,59,2016-11-11
-session,1,3,Internet Explorer 10,28,2017-04-29
-session,1,4,Chrome 13,116,2016-12-28
-user,2,Gregory,Santos,86
-session,2,0,Chrome 35,6,2018-09-21
-session,2,1,Safari 49,85,2017-05-22
-session,2,2,Firefox 47,17,2018-02-02
-session,2,3,Chrome 20,84,2016-11-25
-')
-  end
-
-  def test_result
-    work
-    expected_result = JSON.parse('{"totalUsers":3,"uniqueBrowsersCount":14,"totalSessions":15,"allBrowsers":"CHROME 13,CHROME 20,CHROME 35,CHROME 6,FIREFOX 12,FIREFOX 32,FIREFOX 47,INTERNET EXPLORER 10,INTERNET EXPLORER 28,INTERNET EXPLORER 35,SAFARI 17,SAFARI 29,SAFARI 39,SAFARI 49","usersStats":{"Leida Cira":{"sessionsCount":6,"totalTime":"455 min.","longestSession":"118 min.","browsers":"FIREFOX 12, INTERNET EXPLORER 28, INTERNET EXPLORER 28, INTERNET EXPLORER 35, SAFARI 29, SAFARI 39","usedIE":true,"alwaysUsedChrome":false,"dates":["2017-09-27","2017-03-28","2017-02-27","2016-10-23","2016-09-15","2016-09-01"]},"Palmer Katrina":{"sessionsCount":5,"totalTime":"218 min.","longestSession":"116 min.","browsers":"CHROME 13, CHROME 6, FIREFOX 32, INTERNET EXPLORER 10, SAFARI 17","usedIE":true,"alwaysUsedChrome":false,"dates":["2017-04-29","2016-12-28","2016-12-20","2016-11-11","2016-10-21"]},"Gregory Santos":{"sessionsCount":4,"totalTime":"192 min.","longestSession":"85 min.","browsers":"CHROME 20, CHROME 35, FIREFOX 47, SAFARI 49","usedIE":false,"alwaysUsedChrome":false,"dates":["2018-09-21","2018-02-02","2017-05-22","2016-11-25"]}}}')
-    assert_equal expected_result, JSON.parse(File.read('result.json'))
-  end
-end