fix the bug where the difference between the bulk size * loop and the…

… overlapping hits was not taken, improve loop handling and error prevention

Classes/Command/IndexCommand.php

@@ -30,6 +30,13 @@
 use TYPO3\CMS\Core\Site\SiteFinder;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 
+
+/* ToDo:
+- exception handling (for networking problems, api down etc.?) and check empty zoteroAPIKey, zoteroGroupId
+- sfae the "Last-Modified-Version" Header from zotero and check if the data has changed since the last fetch
+- potential infinite loops, exit strategy with loop count in try/catch
+- fetch the api in bulks of 50 and save bulk wise in elasticsearch?
+*/
 class IndexCommand extends Command
 {
 
@@ -136,22 +143,31 @@ protected function fetchBibliography(): void
  $this->total = 1;
  $cursor = 0;
  // fetch bibliography items bulkwise
- while ($cursor < $this->total) {
+ while ($cursor < ($this->total + $this->bulkSize)) {
  $response = $client->
  group($this->extConf['zoteroGroupId'])->
  items()->
  top()->
  start($cursor)->
  limit($this->bulkSize)->
  send();
- $this->total = (int)$response->getHeaders()['Total-Results'][0];
+ $headers = $response->getHeaders();
+ if (isset($headers['Total-Results'][0])) {
+ $this->total = (int)$headers['Total-Results'][0];
+ } else {
+ $this->io->error('break fetchBibliography loop because of missing Total-Results, ' .$this->total);
+ break; // break the loop if there is no result header to prevent infinite loops
+ }
  if ($cursor === 0) {
  $this->io->progressStart($this->total);
  }
  $collection = new Collection($response->getBody());
  $this->bibliographyItems = $this->bibliographyItems->
  concat($collection->pluck('data'));
- $this->io->progressAdvance($this->bulkSize);
+ // adjust progress bar
+ $remainingItems = $this->total - $cursor;
+ $advanceBy = min($remainingItems, $this->bulkSize);
+ $this->io->progressAdvance($advanceBy);
  $cursor += $this->bulkSize;
  }
  $this->io->progressFinish();
@@ -173,7 +189,7 @@ protected function fetchCitationLocale(string $locale): void
  $result = new Collection();
  $cursor = 0;
  // fetch bibliography items bulkwise
- while ($cursor < $this->total) {
+ while ($cursor < ($this->total + $this->bulkSize)) {
  if ($cursor === 0) {
  $this->io->progressStart($this->total);
  }
@@ -190,7 +206,10 @@ protected function fetchCitationLocale(string $locale): void
  setLocale($locale)->
  send();
  $result = $result->merge(Collection::wrap($response->getBody())->keyBy('key'));
- $this->io->progressAdvance($this->bulkSize);
+ // adjust progress bar
+ $remainingItems = $this->total - $cursor;
+ $advanceBy = min($remainingItems, $this->bulkSize);
+ $this->io->progressAdvance($advanceBy);
  $cursor += $this->bulkSize;
  } catch (\Exception $e) {
  $this->io->newline(2);
@@ -210,7 +229,7 @@ protected function fetchTeiData(): void
  $this->teiDataSets = new Collection();
  $cursor = 0;
  // fetch bibliography items bulkwise
- while ($cursor < $this->total) {
+ while ($cursor < ($this->total + $this->bulkSize)) {
  if ($cursor === 0) {
  $this->io->progressStart($this->total);
  }
@@ -226,7 +245,10 @@ protected function fetchTeiData(): void
  $collection = new Collection($response->getBody());
  $this->teiDataSets = $this->teiDataSets->
  concat($collection->keyBy('key'));
- $this->io->progressAdvance($this->bulkSize);
+ // adjust progress bar
+ $remainingItems = $this->total - $cursor;
+ $advanceBy = min($remainingItems, $this->bulkSize);
+ $this->io->progressAdvance($advanceBy);
  $cursor += $this->bulkSize;
  } catch (\Exception $e) {
  $this->io->newline(2);