Suggested fix for #160

.github/workflows/gradle.yml

@@ -10,11 +10,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
-      uses: actions/setup-java@v1
+    - uses: actions/checkout@v3
+    - name: Set up JDK 8
+      uses: actions/setup-java@v3
       with:
-        java-version: 1.8
+        distribution: 'temurin'
+        java-version: 8
 
     - name: build and test cadc-jsqlparser-compat
       run: cd cadc-jsqlparser-compat && ../gradlew --info clean build javadoc install

cadc-tap-server/build.gradle

@@ -13,24 +13,25 @@ sourceCompatibility = 1.8
 
 group = 'org.opencadc'
 
-version = '1.1.23'
+version = '1.2.0'
 
 description = 'OpenCADC TAP-1.1 tap server library'
 def git_url = 'https://github.com/opencadc/tap'
 
 dependencies {
-    compile 'org.jdom:jdom2:[2.0,)'
-
-    compile 'org.opencadc:cadc-util:[1.6,)'
-    compile 'org.opencadc:cadc-tap:[1.1.0,1.2)'
-    compile 'org.opencadc:cadc-tap-schema:[1.1.28,1.2)'
-    compile 'org.opencadc:cadc-vosi:[1.4.2,)'
-    compile 'org.opencadc:cadc-dali:[1.2.5,)'
-    compile 'org.opencadc:cadc-registry:[1.4,)'
-    compile 'org.opencadc:cadc-uws:[1.0,)'
-    compile 'org.opencadc:cadc-uws-server:[1.1,)'
-
-    testCompile 'junit:junit:[4.0,5.0)'
-    testCompile 'xerces:xercesImpl:[2.0,)'
-    testCompile 'org.jdom:jaxen-jdom:1.0-FCS'
+    implementation 'org.jdom:jdom2:[2.0,)'
+
+    implementation 'org.opencadc:cadc-util:[1.6,)'
+    implementation 'org.opencadc:cadc-tap:[1.1.0,1.2)'
+    implementation 'org.opencadc:cadc-tap-schema:[1.1.28,1.2)'
+    implementation 'org.opencadc:cadc-vosi:[1.4.2,)'
+    implementation 'org.opencadc:cadc-dali:[1.2.5,)'
+    implementation 'org.opencadc:cadc-registry:[1.4,)'
+    implementation 'org.opencadc:cadc-uws:[1.0,)'
+    implementation 'org.opencadc:cadc-uws-server:[1.1,)'
+
+    testImplementation 'junit:junit:[4.0,5.0)'
+    testImplementation 'xerces:xercesImpl:[2.0,)'
+    testImplementation 'org.jdom:jaxen-jdom:1.0-FCS'
+    testImplementation 'org.apache.commons:commons-lang3:[3.14.0, )'
 }

cadc-tap-server/src/main/java/ca/nrc/cadc/tap/BasicUploadManager.java

@@ -1,4 +1,4 @@
-/*
+ /*
 ************************************************************************
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
@@ -77,6 +77,7 @@
 import ca.nrc.cadc.tap.schema.ColumnDesc;
 import ca.nrc.cadc.tap.schema.TableDesc;
 import ca.nrc.cadc.tap.upload.JDOMVOTableParser;
+import ca.nrc.cadc.tap.upload.UploadLimits;
 import ca.nrc.cadc.tap.upload.UploadParameters;
 import ca.nrc.cadc.tap.upload.UploadTable;
 import ca.nrc.cadc.tap.upload.VOTableParser;
@@ -101,10 +102,7 @@
 public class BasicUploadManager implements UploadManager
 {
     private static final Logger log = Logger.getLogger(BasicUploadManager.class);
-
-    // Number of rows to insert per commit.
-    private static final int NUM_ROWS_PER_COMMIT = 100;
-
+
     /**
      * DataSource for the DB.
      */
@@ -121,9 +119,9 @@ public class BasicUploadManager implements UploadManager
     protected DateFormat dateFormat;
 
     /**
-     * Maximum number of rows allowed in the UPLOAD VOTable.
+     * Limitations on the UPLOAD VOTable.  Defaults to one Megabyte with unlimited rows and columns.
      */
-    protected int maxUploadRows;
+    protected UploadLimits uploadLimits = new UploadLimits(1024L * 1024L);
 
     protected Job job;
 
@@ -132,9 +130,13 @@ public class BasicUploadManager implements UploadManager
      */
     private BasicUploadManager() { }
 
-    protected BasicUploadManager(int maxUploadRows)
+    protected BasicUploadManager(UploadLimits uploadLimits)
     {
-        this.maxUploadRows = maxUploadRows;
+        if (uploadLimits == null) {
+            throw new IllegalStateException("Upload limits are required.");
+        }
+
+        this.uploadLimits = uploadLimits;
         dateFormat = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC);
     }
 
@@ -265,9 +267,9 @@ public TableDesc acceptTargetTableDesc(TableDesc td) {
     }
 
     protected VOTableParser getVOTableParser(UploadTable uploadTable)
-            throws IOException
+            throws VOTableParserException
     {
-        VOTableParser ret = new JDOMVOTableParser();
+        VOTableParser ret = new JDOMVOTableParser(uploadLimits);
         ret.setUpload(uploadTable);
         return ret;
     }

cadc-tap-server/src/main/java/ca/nrc/cadc/tap/upload/JDOMVOTableParser.java

@@ -70,22 +70,23 @@
 package ca.nrc.cadc.tap.upload;
 
 import ca.nrc.cadc.dali.tables.votable.VOTableDocument;
-import ca.nrc.cadc.dali.tables.votable.VOTableField;
 import ca.nrc.cadc.dali.tables.votable.VOTableReader;
 import ca.nrc.cadc.dali.tables.votable.VOTableResource;
 import ca.nrc.cadc.dali.tables.votable.VOTableTable;
+import ca.nrc.cadc.io.ByteCountInputStream;
+import ca.nrc.cadc.io.ByteLimitExceededException;
 import ca.nrc.cadc.tap.UploadManager;
 import static ca.nrc.cadc.tap.UploadManager.SCHEMA;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
 import org.apache.log4j.Logger;
-import ca.nrc.cadc.tap.schema.ColumnDesc;
 import ca.nrc.cadc.tap.schema.TableDesc;
 import ca.nrc.cadc.tap.schema.TapSchemaUtil;
 import java.io.IOException;
 
+
 /**
  * Implements the VOTableParser interface using JDOM.
  *
@@ -99,8 +100,19 @@ public class JDOMVOTableParser implements VOTableParser
 
     protected String tableName;
     protected VOTableTable votable;
+    protected final UploadLimits uploadLimits;
+
 
-    public JDOMVOTableParser() { }
+    /**
+     * Constructor setting limits on upload tables.
+     * @param uploadLimits  Limitations of the Upload table.  Required.
+     */
+    public JDOMVOTableParser(UploadLimits uploadLimits) {
+        if (uploadLimits == null) {
+            throw new IllegalStateException("Upload limits are required.");
+        }
+        this.uploadLimits = uploadLimits;
+    }
 
     @Override
     public void setUpload(UploadTable upload)
@@ -113,28 +125,64 @@ private void init()
     {
         if (votable == null)
         {
-            VOTableReader r = new VOTableReader();
-            // TODO: wrap a ByteCountInputStream here to limit input data volume
-            VOTableDocument doc = r.read(upload.uri.toURL().openStream());
+            VOTableDocument doc = verifyUploadTable();
             VOTableResource vr = doc.getResourceByType("results");
             this.votable = vr.getTable();
-
             this.tableName = upload.tableName;
-            if (!tableName.toUpperCase().startsWith(SCHEMA))
+            if (!tableName.toUpperCase().startsWith(SCHEMA)) {
                 tableName = SCHEMA + "." + tableName;
+            }
+        }
+    }
+
+    /**
+     * Ensure the Upload table conforms to specified limitations, if any.  This will only read through the file if
+     * the Upload table file falls into the acceptable size first.
+     * @throws IOException  If the file cannot be read, or if the URI to the Upload file is invalid.
+     */
+    VOTableDocument verifyUploadTable() throws IOException {
+        // Only proceed if a size limitation is set.
+        final VOTableReader voTableReader = new VOTableReader();
+        try (final ByteCountInputStream byteCountInputStream =
+                     new ByteCountInputStream(upload.uri.toURL().openStream(), uploadLimits.byteLimit)) {
+            final VOTableDocument doc = voTableReader.read(byteCountInputStream);
+            final VOTableResource vr = doc.getResourceByType("results");
+            final VOTableTable voTableTable = vr.getTable();
+
+            if (uploadLimits.columnLimit != null && voTableTable.getFields().size() > uploadLimits.columnLimit) {
+                throw new IllegalArgumentException("Column count exceeds maximum of " + uploadLimits.columnLimit);
+            }
+
+            // Avoid iterating if no row limit has been set.
+            if (uploadLimits.rowLimit != null) {
+                int counter = 0;
+                for (final Iterator<List<Object>> iterator = voTableTable.getTableData().iterator();
+                     iterator.hasNext();) {
+                    if (++counter > uploadLimits.rowLimit) {
+                        throw new IllegalArgumentException("Row count exceeds maximum of " + uploadLimits.rowLimit);
+                    } else {
+                        iterator.next();
+                    }
+                }
+            }
+
+            return doc;
+        } catch (ByteLimitExceededException byteLimitExceededException) {
+            throw new IllegalArgumentException("Size of upload file exceeds maximum of "
+                                               + byteLimitExceededException.getLimit() + " bytes.",
+                                               byteLimitExceededException);
         }
     }
 
     /**
      * Get a List that describes each VOTable column.
      *
-     * @throws java.io.IOException
-     * @throws VOTableParserException if unable to parse the VOTable.
+     * @throws java.io.IOException if unable to parse the VOTable.
      * @return List of ColumnDesc describing the VOTable columns.
      */
     @Override
     public TableDesc getTableDesc()
-        throws IOException, VOTableParserException
+        throws IOException
     {
         init();
         TableDesc tableDesc = TapSchemaUtil.createTableDesc(UploadManager.SCHEMA, tableName, votable);

cadc-tap-server/src/main/java/ca/nrc/cadc/tap/upload/UploadLimits.java

@@ -0,0 +1,83 @@
+/*
+ ************************************************************************
+ *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
+ **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
+ *
+ *  (c) 2024.                            (c) 2024.
+ *  Government of Canada                 Gouvernement du Canada
+ *  National Research Council            Conseil national de recherches
+ *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
+ *  All rights reserved                  Tous droits réservés
+ *
+ *  NRC disclaims any warranties,        Le CNRC dénie toute garantie
+ *  expressed, implied, or               énoncée, implicite ou légale,
+ *  statutory, of any kind with          de quelque nature que ce
+ *  respect to the software,             soit, concernant le logiciel,
+ *  including without limitation         y compris sans restriction
+ *  any warranty of merchantability      toute garantie de valeur
+ *  or fitness for a particular          marchande ou de pertinence
+ *  purpose. NRC shall not be            pour un usage particulier.
+ *  liable in any event for any          Le CNRC ne pourra en aucun cas
+ *  damages, whether direct or           être tenu responsable de tout
+ *  indirect, special or general,        dommage, direct ou indirect,
+ *  consequential or incidental,         particulier ou général,
+ *  arising from the use of the          accessoire ou fortuit, résultant
+ *  software.  Neither the name          de l'utilisation du logiciel. Ni
+ *  of the National Research             le nom du Conseil National de
+ *  Council of Canada nor the            Recherches du Canada ni les noms
+ *  names of its contributors may        de ses  participants ne peuvent
+ *  be used to endorse or promote        être utilisés pour approuver ou
+ *  products derived from this           promouvoir les produits dérivés
+ *  software without specific prior      de ce logiciel sans autorisation
+ *  written permission.                  préalable et particulière
+ *                                       par écrit.
+ *
+ *  This file is part of the             Ce fichier fait partie du projet
+ *  OpenCADC project.                    OpenCADC.
+ *
+ *  OpenCADC is free software:           OpenCADC est un logiciel libre ;
+ *  you can redistribute it and/or       vous pouvez le redistribuer ou le
+ *  modify it under the terms of         modifier suivant les termes de
+ *  the GNU Affero General Public        la “GNU Affero General Public
+ *  License as published by the          License” telle que publiée
+ *  Free Software Foundation,            par la Free Software Foundation
+ *  either version 3 of the              : soit la version 3 de cette
+ *  License, or (at your option)         licence, soit (à votre gré)
+ *  any later version.                   toute version ultérieure.
+ *
+ *  OpenCADC is distributed in the       OpenCADC est distribué
+ *  hope that it will be useful,         dans l’espoir qu’il vous
+ *  but WITHOUT ANY WARRANTY;            sera utile, mais SANS AUCUNE
+ *  without even the implied             GARANTIE : sans même la garantie
+ *  warranty of MERCHANTABILITY          implicite de COMMERCIALISABILITÉ
+ *  or FITNESS FOR A PARTICULAR          ni d’ADÉQUATION À UN OBJECTIF
+ *  PURPOSE.  See the GNU Affero         PARTICULIER. Consultez la Licence
+ *  General Public License for           Générale Publique GNU Affero
+ *  more details.                        pour plus de détails.
+ *
+ *  You should have received             Vous devriez avoir reçu une
+ *  a copy of the GNU Affero             copie de la Licence Générale
+ *  General Public License along         Publique GNU Affero avec
+ *  with OpenCADC.  If not, see          OpenCADC ; si ce n’est
+ *  <http://www.gnu.org/licenses/>.      pas le cas, consultez :
+ *                                       <http://www.gnu.org/licenses/>.
+ *
+ *
+ ************************************************************************
+ */
+
+package ca.nrc.cadc.tap.upload;
+
+public class UploadLimits {
+    final long byteLimit;
+    public Integer rowLimit;
+    public Integer columnLimit;
+
+    public UploadLimits(long byteLimit) {
+        if (byteLimit <= 0) {
+            throw new IllegalStateException(
+                    "Invalid configuration: Upload file size limit should be greater than zero.");
+        }
+        this.byteLimit = byteLimit;
+    }
+}

cadc-tap-server/src/main/java/ca/nrc/cadc/tap/upload/VOTableParser.java

@@ -82,9 +82,12 @@
  */
 public interface VOTableParser
 {
-    public static final String TAP_SCHEMA = "tap_schema";
-
-    void setUpload(UploadTable upload);
+    /**
+     * Set the Upload Table details.
+     * @param upload        The upload table descriptor.
+     * @throws VOTableParserException   For any parsing exceptions.
+     */
+    void setUpload(UploadTable upload) throws VOTableParserException;
 
     /**
      * Get a TableDesc of the VOTable.