001/*-
002 * #%L
003 * HAPI FHIR Server - SQL Migration
004 * %%
005 * Copyright (C) 2014 - 2026 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.migrate.taskdef;
021
022import ca.uhn.fhir.i18n.Msg;
023import ca.uhn.fhir.util.StopWatch;
024import ca.uhn.fhir.util.VersionEnum;
025import com.google.common.collect.ForwardingMap;
026import org.apache.commons.lang3.concurrent.BasicThreadFactory;
027import org.slf4j.Logger;
028import org.slf4j.LoggerFactory;
029import org.springframework.jdbc.core.ColumnMapRowMapper;
030import org.springframework.jdbc.core.JdbcTemplate;
031import org.springframework.jdbc.core.RowCallbackHandler;
032
033import java.sql.ResultSet;
034import java.sql.SQLException;
035import java.util.ArrayList;
036import java.util.Date;
037import java.util.HashMap;
038import java.util.List;
039import java.util.Map;
040import java.util.concurrent.Future;
041import java.util.concurrent.LinkedBlockingQueue;
042import java.util.concurrent.RejectedExecutionException;
043import java.util.concurrent.RejectedExecutionHandler;
044import java.util.concurrent.ThreadPoolExecutor;
045import java.util.concurrent.TimeUnit;
046import java.util.function.Function;
047
048public abstract class BaseColumnCalculatorTask extends BaseTableColumnTask {
049
050        protected static final Logger ourLog = LoggerFactory.getLogger(BaseColumnCalculatorTask.class);
051        private int myBatchSize = 10000;
052        private ThreadPoolExecutor myExecutor;
053        private String myPidColumnName;
054
055        /**
056         * Constructor
057         */
058        public BaseColumnCalculatorTask(VersionEnum theRelease, String theVersion) {
059                this(theRelease.toString(), theVersion);
060        }
061
062        /**
063         * Constructor
064         */
065        public BaseColumnCalculatorTask(String theRelease, String theVersion) {
066                super(theRelease, theVersion);
067        }
068
069        public void setBatchSize(int theBatchSize) {
070                myBatchSize = theBatchSize;
071        }
072
073        /**
074         * Allows concrete implementations to decide if they should be skipped.
075         *
076         * @return a boolean indicating whether or not to skip execution of the task.
077         */
078        protected abstract boolean shouldSkipTask();
079
080        @Override
081        public synchronized void doExecute() throws SQLException {
082                if (isDryRun() || shouldSkipTask()) {
083                        return;
084                }
085
086                initializeExecutor();
087
088                try {
089
090                        while (true) {
091                                MyRowCallbackHandler rch = new MyRowCallbackHandler();
092                                getTxTemplate().execute(t -> {
093                                        JdbcTemplate jdbcTemplate = newJdbcTemplate();
094                                        jdbcTemplate.setMaxRows(100000);
095
096                                        String sql = "SELECT * FROM " + getTableName() + " WHERE " + getWhereClause();
097                                        logInfo(
098                                                        ourLog,
099                                                        "Finding up to {} rows in {} that requires calculations, using query: {}",
100                                                        myBatchSize,
101                                                        getTableName(),
102                                                        sql);
103
104                                        jdbcTemplate.query(sql, rch);
105                                        rch.done();
106
107                                        return null;
108                                });
109
110                                rch.submitNext();
111                                List<Future<?>> futures = rch.getFutures();
112                                if (futures.isEmpty()) {
113                                        break;
114                                }
115
116                                logInfo(ourLog, "Waiting for {} tasks to complete", futures.size());
117                                for (Future<?> next : futures) {
118                                        try {
119                                                next.get();
120                                        } catch (Exception e) {
121                                                throw new SQLException(Msg.code(69) + e, e);
122                                        }
123                                }
124                        }
125
126                } finally {
127                        destroyExecutor();
128                }
129        }
130
131        private void destroyExecutor() {
132                myExecutor.shutdownNow();
133        }
134
135        private void initializeExecutor() {
136                int maximumPoolSize = Runtime.getRuntime().availableProcessors();
137
138                LinkedBlockingQueue<Runnable> executorQueue = new LinkedBlockingQueue<>(maximumPoolSize);
139                BasicThreadFactory threadFactory = new BasicThreadFactory.Builder()
140                                .namingPattern("worker-" + "-%d")
141                                .daemon(false)
142                                .priority(Thread.NORM_PRIORITY)
143                                .build();
144                RejectedExecutionHandler rejectedExecutionHandler = new RejectedExecutionHandler() {
145                        @Override
146                        public void rejectedExecution(Runnable theRunnable, ThreadPoolExecutor theExecutor) {
147                                logInfo(
148                                                ourLog,
149                                                "Note: Executor queue is full ({} elements), waiting for a slot to become available!",
150                                                executorQueue.size());
151                                StopWatch sw = new StopWatch();
152                                try {
153                                        executorQueue.put(theRunnable);
154                                } catch (InterruptedException theE) {
155                                        throw new RejectedExecutionException(
156                                                        Msg.code(70) + "Task " + theRunnable.toString() + " rejected from " + theE.toString());
157                                }
158                                logInfo(ourLog, "Slot become available after {}ms", sw.getMillis());
159                        }
160                };
161                myExecutor = new ThreadPoolExecutor(
162                                maximumPoolSize,
163                                maximumPoolSize,
164                                0L,
165                                TimeUnit.MILLISECONDS,
166                                executorQueue,
167                                threadFactory,
168                                rejectedExecutionHandler);
169        }
170
171        public BaseColumnCalculatorTask setPidColumnName(String thePidColumnName) {
172                myPidColumnName = thePidColumnName;
173                return this;
174        }
175
176        private Future<?> updateRows(List<Map<String, Object>> theRows) {
177                Runnable task = () -> {
178                        StopWatch sw = new StopWatch();
179                        getTxTemplate().execute(t -> {
180
181                                // Loop through rows
182                                assert theRows != null;
183                                for (Map<String, Object> nextRow : theRows) {
184
185                                        Map<String, Object> newValues = new HashMap<>();
186                                        MandatoryKeyMap<String, Object> nextRowMandatoryKeyMap = new MandatoryKeyMap<>(nextRow);
187
188                                        // Apply calculators
189                                        for (Map.Entry<String, Function<MandatoryKeyMap<String, Object>, Object>> nextCalculatorEntry :
190                                                        myCalculators.entrySet()) {
191                                                String nextColumn = nextCalculatorEntry.getKey();
192                                                Function<MandatoryKeyMap<String, Object>, Object> nextCalculator =
193                                                                nextCalculatorEntry.getValue();
194                                                Object value = nextCalculator.apply(nextRowMandatoryKeyMap);
195                                                newValues.put(nextColumn, value);
196                                        }
197
198                                        // Generate update SQL
199                                        StringBuilder sqlBuilder = new StringBuilder();
200                                        List<Object> arguments = new ArrayList<>();
201                                        sqlBuilder.append("UPDATE ");
202                                        sqlBuilder.append(getTableName());
203                                        sqlBuilder.append(" SET ");
204                                        for (Map.Entry<String, Object> nextNewValueEntry : newValues.entrySet()) {
205                                                if (arguments.size() > 0) {
206                                                        sqlBuilder.append(", ");
207                                                }
208                                                sqlBuilder.append(nextNewValueEntry.getKey()).append(" = ?");
209                                                arguments.add(nextNewValueEntry.getValue());
210                                        }
211                                        sqlBuilder.append(" WHERE " + myPidColumnName + " = ?");
212                                        arguments.add((Number) nextRow.get(myPidColumnName));
213
214                                        // Apply update SQL
215                                        newJdbcTemplate().update(sqlBuilder.toString(), arguments.toArray());
216                                }
217                                return theRows.size();
218                        });
219                        logInfo(ourLog, "Updated {} rows on {} in {}", theRows.size(), getTableName(), sw.toString());
220                };
221                return myExecutor.submit(task);
222        }
223
224        public static class MandatoryKeyMap<K, V> extends ForwardingMap<K, V> {
225
226                private final Map<K, V> myWrap;
227
228                public MandatoryKeyMap(Map<K, V> theWrap) {
229                        myWrap = theWrap;
230                }
231
232                @Override
233                public V get(Object theKey) {
234                        if (!containsKey(theKey)) {
235                                throw new IllegalArgumentException(Msg.code(71) + "No key: " + theKey);
236                        }
237                        return super.get(theKey);
238                }
239
240                public String getString(String theKey) {
241                        return (String) get(theKey);
242                }
243
244                public Date getDate(String theKey) {
245                        return (Date) get(theKey);
246                }
247
248                @Override
249                protected Map<K, V> delegate() {
250                        return myWrap;
251                }
252
253                public String getResourceType() {
254                        return getString("RES_TYPE");
255                }
256
257                public String getParamName() {
258                        return getString("SP_NAME");
259                }
260        }
261
262        private class MyRowCallbackHandler implements RowCallbackHandler {
263
264                private List<Map<String, Object>> myRows = new ArrayList<>();
265                private List<Future<?>> myFutures = new ArrayList<>();
266
267                @Override
268                public void processRow(ResultSet rs) throws SQLException {
269                        Map<String, Object> row = new ColumnMapRowMapper().mapRow(rs, 0);
270                        myRows.add(row);
271
272                        if (myRows.size() >= myBatchSize) {
273                                submitNext();
274                        }
275                }
276
277                private void submitNext() {
278                        if (myRows.size() > 0) {
279                                myFutures.add(updateRows(myRows));
280                                myRows = new ArrayList<>();
281                        }
282                }
283
284                public List<Future<?>> getFutures() {
285                        return myFutures;
286                }
287
288                public void done() {
289                        if (myRows.size() > 0) {
290                                submitNext();
291                        }
292                }
293        }
294}