001/*- 002 * #%L 003 * HAPI FHIR Server - SQL Migration 004 * %% 005 * Copyright (C) 2014 - 2025 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.migrate.taskdef; 021 022import ca.uhn.fhir.i18n.Msg; 023import ca.uhn.fhir.util.StopWatch; 024import ca.uhn.fhir.util.VersionEnum; 025import com.google.common.collect.ForwardingMap; 026import org.apache.commons.lang3.concurrent.BasicThreadFactory; 027import org.slf4j.Logger; 028import org.slf4j.LoggerFactory; 029import org.springframework.jdbc.core.ColumnMapRowMapper; 030import org.springframework.jdbc.core.JdbcTemplate; 031import org.springframework.jdbc.core.RowCallbackHandler; 032 033import java.sql.ResultSet; 034import java.sql.SQLException; 035import java.util.ArrayList; 036import java.util.Date; 037import java.util.HashMap; 038import java.util.List; 039import java.util.Map; 040import java.util.concurrent.Future; 041import java.util.concurrent.LinkedBlockingQueue; 042import java.util.concurrent.RejectedExecutionException; 043import java.util.concurrent.RejectedExecutionHandler; 044import java.util.concurrent.ThreadPoolExecutor; 045import java.util.concurrent.TimeUnit; 046import java.util.function.Function; 047 048public abstract class BaseColumnCalculatorTask extends BaseTableColumnTask { 049 050 protected static final Logger ourLog = LoggerFactory.getLogger(BaseColumnCalculatorTask.class); 051 private int myBatchSize = 10000; 052 private ThreadPoolExecutor myExecutor; 053 private String myPidColumnName; 054 055 /** 056 * Constructor 057 */ 058 public BaseColumnCalculatorTask(VersionEnum theRelease, String theVersion) { 059 this(theRelease.toString(), theVersion); 060 } 061 062 /** 063 * Constructor 064 */ 065 public BaseColumnCalculatorTask(String theRelease, String theVersion) { 066 super(theRelease, theVersion); 067 } 068 069 public void setBatchSize(int theBatchSize) { 070 myBatchSize = theBatchSize; 071 } 072 073 /** 074 * Allows concrete implementations to decide if they should be skipped. 075 * 076 * @return a boolean indicating whether or not to skip execution of the task. 077 */ 078 protected abstract boolean shouldSkipTask(); 079 080 @Override 081 public synchronized void doExecute() throws SQLException { 082 if (isDryRun() || shouldSkipTask()) { 083 return; 084 } 085 086 initializeExecutor(); 087 088 try { 089 090 while (true) { 091 MyRowCallbackHandler rch = new MyRowCallbackHandler(); 092 getTxTemplate().execute(t -> { 093 JdbcTemplate jdbcTemplate = newJdbcTemplate(); 094 jdbcTemplate.setMaxRows(100000); 095 096 String sql = "SELECT * FROM " + getTableName() + " WHERE " + getWhereClause(); 097 logInfo( 098 ourLog, 099 "Finding up to {} rows in {} that requires calculations, using query: {}", 100 myBatchSize, 101 getTableName(), 102 sql); 103 104 jdbcTemplate.query(sql, rch); 105 rch.done(); 106 107 return null; 108 }); 109 110 rch.submitNext(); 111 List<Future<?>> futures = rch.getFutures(); 112 if (futures.isEmpty()) { 113 break; 114 } 115 116 logInfo(ourLog, "Waiting for {} tasks to complete", futures.size()); 117 for (Future<?> next : futures) { 118 try { 119 next.get(); 120 } catch (Exception e) { 121 throw new SQLException(Msg.code(69) + e, e); 122 } 123 } 124 } 125 126 } finally { 127 destroyExecutor(); 128 } 129 } 130 131 private void destroyExecutor() { 132 myExecutor.shutdownNow(); 133 } 134 135 private void initializeExecutor() { 136 int maximumPoolSize = Runtime.getRuntime().availableProcessors(); 137 138 LinkedBlockingQueue<Runnable> executorQueue = new LinkedBlockingQueue<>(maximumPoolSize); 139 BasicThreadFactory threadFactory = new BasicThreadFactory.Builder() 140 .namingPattern("worker-" + "-%d") 141 .daemon(false) 142 .priority(Thread.NORM_PRIORITY) 143 .build(); 144 RejectedExecutionHandler rejectedExecutionHandler = new RejectedExecutionHandler() { 145 @Override 146 public void rejectedExecution(Runnable theRunnable, ThreadPoolExecutor theExecutor) { 147 logInfo( 148 ourLog, 149 "Note: Executor queue is full ({} elements), waiting for a slot to become available!", 150 executorQueue.size()); 151 StopWatch sw = new StopWatch(); 152 try { 153 executorQueue.put(theRunnable); 154 } catch (InterruptedException theE) { 155 throw new RejectedExecutionException( 156 Msg.code(70) + "Task " + theRunnable.toString() + " rejected from " + theE.toString()); 157 } 158 logInfo(ourLog, "Slot become available after {}ms", sw.getMillis()); 159 } 160 }; 161 myExecutor = new ThreadPoolExecutor( 162 maximumPoolSize, 163 maximumPoolSize, 164 0L, 165 TimeUnit.MILLISECONDS, 166 executorQueue, 167 threadFactory, 168 rejectedExecutionHandler); 169 } 170 171 public BaseColumnCalculatorTask setPidColumnName(String thePidColumnName) { 172 myPidColumnName = thePidColumnName; 173 return this; 174 } 175 176 private Future<?> updateRows(List<Map<String, Object>> theRows) { 177 Runnable task = () -> { 178 StopWatch sw = new StopWatch(); 179 getTxTemplate().execute(t -> { 180 181 // Loop through rows 182 assert theRows != null; 183 for (Map<String, Object> nextRow : theRows) { 184 185 Map<String, Object> newValues = new HashMap<>(); 186 MandatoryKeyMap<String, Object> nextRowMandatoryKeyMap = new MandatoryKeyMap<>(nextRow); 187 188 // Apply calculators 189 for (Map.Entry<String, Function<MandatoryKeyMap<String, Object>, Object>> nextCalculatorEntry : 190 myCalculators.entrySet()) { 191 String nextColumn = nextCalculatorEntry.getKey(); 192 Function<MandatoryKeyMap<String, Object>, Object> nextCalculator = 193 nextCalculatorEntry.getValue(); 194 Object value = nextCalculator.apply(nextRowMandatoryKeyMap); 195 newValues.put(nextColumn, value); 196 } 197 198 // Generate update SQL 199 StringBuilder sqlBuilder = new StringBuilder(); 200 List<Object> arguments = new ArrayList<>(); 201 sqlBuilder.append("UPDATE "); 202 sqlBuilder.append(getTableName()); 203 sqlBuilder.append(" SET "); 204 for (Map.Entry<String, Object> nextNewValueEntry : newValues.entrySet()) { 205 if (arguments.size() > 0) { 206 sqlBuilder.append(", "); 207 } 208 sqlBuilder.append(nextNewValueEntry.getKey()).append(" = ?"); 209 arguments.add(nextNewValueEntry.getValue()); 210 } 211 sqlBuilder.append(" WHERE " + myPidColumnName + " = ?"); 212 arguments.add((Number) nextRow.get(myPidColumnName)); 213 214 // Apply update SQL 215 newJdbcTemplate().update(sqlBuilder.toString(), arguments.toArray()); 216 } 217 return theRows.size(); 218 }); 219 logInfo(ourLog, "Updated {} rows on {} in {}", theRows.size(), getTableName(), sw.toString()); 220 }; 221 return myExecutor.submit(task); 222 } 223 224 public static class MandatoryKeyMap<K, V> extends ForwardingMap<K, V> { 225 226 private final Map<K, V> myWrap; 227 228 public MandatoryKeyMap(Map<K, V> theWrap) { 229 myWrap = theWrap; 230 } 231 232 @Override 233 public V get(Object theKey) { 234 if (!containsKey(theKey)) { 235 throw new IllegalArgumentException(Msg.code(71) + "No key: " + theKey); 236 } 237 return super.get(theKey); 238 } 239 240 public String getString(String theKey) { 241 return (String) get(theKey); 242 } 243 244 public Date getDate(String theKey) { 245 return (Date) get(theKey); 246 } 247 248 @Override 249 protected Map<K, V> delegate() { 250 return myWrap; 251 } 252 253 public String getResourceType() { 254 return getString("RES_TYPE"); 255 } 256 257 public String getParamName() { 258 return getString("SP_NAME"); 259 } 260 } 261 262 private class MyRowCallbackHandler implements RowCallbackHandler { 263 264 private List<Map<String, Object>> myRows = new ArrayList<>(); 265 private List<Future<?>> myFutures = new ArrayList<>(); 266 267 @Override 268 public void processRow(ResultSet rs) throws SQLException { 269 Map<String, Object> row = new ColumnMapRowMapper().mapRow(rs, 0); 270 myRows.add(row); 271 272 if (myRows.size() >= myBatchSize) { 273 submitNext(); 274 } 275 } 276 277 private void submitNext() { 278 if (myRows.size() > 0) { 279 myFutures.add(updateRows(myRows)); 280 myRows = new ArrayList<>(); 281 } 282 } 283 284 public List<Future<?>> getFutures() { 285 return myFutures; 286 } 287 288 public void done() { 289 if (myRows.size() > 0) { 290 submitNext(); 291 } 292 } 293 } 294}