Fix issues with tootctl's parallelization and progress reporting (#12093)
This commit is contained in:
		
					parent
					
						
							
								b5be067c88
							
						
					
				
			
			
				commit
				
					
						38b6c34e32
					
				
			
		
					 1 changed files with 24 additions and 10 deletions
				
			
		|  | @ -15,6 +15,11 @@ module Mastodon | |||
|     end | ||||
| 
 | ||||
|     def parallelize_with_progress(scope) | ||||
|       if options[:concurrency] < 1 | ||||
|         say('Cannot run with this concurrency setting, must be at least 1', :red) | ||||
|         exit(1) | ||||
|       end | ||||
| 
 | ||||
|       ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] | ||||
| 
 | ||||
|       progress  = create_progress_bar(scope.count) | ||||
|  | @ -27,17 +32,26 @@ module Mastodon | |||
| 
 | ||||
|         items.each do |item| | ||||
|           futures << Concurrent::Future.execute(executor: pool) do | ||||
|             ActiveRecord::Base.connection_pool.with_connection do | ||||
|               begin | ||||
|                 progress.log("Processing #{item.id}") if options[:verbose] | ||||
|             begin | ||||
|               if !progress.total.nil? && progress.progress + 1 > progress.total | ||||
|                 # The number of items has changed between start and now, | ||||
|                 # since there is no good way to predict the final count from | ||||
|                 # here, just change the progress bar to an indeterminate one | ||||
| 
 | ||||
|                 result = yield(item) | ||||
|                 aggregate.increment(result) if result.is_a?(Integer) | ||||
|               rescue => e | ||||
|                 progress.log pastel.red("Error processing #{item.id}: #{e}") | ||||
|               ensure | ||||
|                 progress.increment | ||||
|                 progress.total = nil | ||||
|               end | ||||
| 
 | ||||
|               progress.log("Processing #{item.id}") if options[:verbose] | ||||
| 
 | ||||
|               result = ActiveRecord::Base.connection_pool.with_connection do | ||||
|                 yield(item) | ||||
|               end | ||||
| 
 | ||||
|               aggregate.increment(result) if result.is_a?(Integer) | ||||
|             rescue => e | ||||
|               progress.log pastel.red("Error processing #{item.id}: #{e}") | ||||
|             ensure | ||||
|               progress.increment | ||||
|             end | ||||
|           end | ||||
|         end | ||||
|  | @ -46,7 +60,7 @@ module Mastodon | |||
|         futures.map(&:value) | ||||
|       end | ||||
| 
 | ||||
|       progress.finish | ||||
|       progress.stop | ||||
| 
 | ||||
|       [total.value, aggregate.value] | ||||
|     end | ||||
|  |  | |||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue