From 8addbaf3fcd9c664139688e2f13192e8e8948ee0 Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Fri, 15 Apr 2022 16:04:13 +1200 Subject: [PATCH] Remove another 'in-between' class As with Contribution & Membership there is a pointless in-between class that is only inherited by one class... --- CRM/Activity/Import/Form/Preview.php | 12 +- CRM/Activity/Import/Form/Summary.php | 4 +- CRM/Activity/Import/Parser.php | 361 ------------------------ CRM/Activity/Import/Parser/Activity.php | 344 +++++++++++++++++++++- 4 files changed, 351 insertions(+), 370 deletions(-) delete mode 100644 CRM/Activity/Import/Parser.php diff --git a/CRM/Activity/Import/Form/Preview.php b/CRM/Activity/Import/Form/Preview.php index 6ce5e0d84987..75cd9db82eff 100644 --- a/CRM/Activity/Import/Form/Preview.php +++ b/CRM/Activity/Import/Form/Preview.php @@ -42,17 +42,17 @@ public function preProcess() { $this->assign('savedMappingName', $mappingId ? $mapDAO->name : NULL); if ($invalidRowCount) { - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Activity_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Activity_Import_Parser_Activity'; $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } if ($conflictRowCount) { - $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Activity_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Activity_Import_Parser_Activity'; $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } if ($mismatchCount) { - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Activity_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Activity_Import_Parser_Activity'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } @@ -137,11 +137,11 @@ public function postProcess() { fclose($fd); $this->set('errorFile', $errorFile); - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Activity_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Activity_Import_Parser_Activity'; $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Activity_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Activity_Import_Parser_Activity'; $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Activity_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Activity_Import_Parser_Activity'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } } diff --git a/CRM/Activity/Import/Form/Summary.php b/CRM/Activity/Import/Form/Summary.php index b8fc8327ab58..e2543665ccb6 100644 --- a/CRM/Activity/Import/Form/Summary.php +++ b/CRM/Activity/Import/Form/Summary.php @@ -38,11 +38,11 @@ public function preProcess() { $onDuplicate = $this->get('onDuplicate'); $mismatchCount = $this->get('unMatchCount'); if ($duplicateRowCount > 0) { - $urlParams = 'type=' . CRM_Import_Parser::DUPLICATE . '&parser=CRM_Activity_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::DUPLICATE . '&parser=CRM_Activity_Import_Parser_Activity'; $this->set('downloadDuplicateRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } elseif ($mismatchCount) { - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Activity_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Activity_Import_Parser_Activity'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } else { diff --git a/CRM/Activity/Import/Parser.php b/CRM/Activity/Import/Parser.php deleted file mode 100644 index 6e4d290bf594..000000000000 --- a/CRM/Activity/Import/Parser.php +++ /dev/null @@ -1,361 +0,0 @@ -init(); - - $this->_haveColumnHeader = $skipColumnHeader; - - $this->_separator = $separator; - - $fd = fopen($fileName, "r"); - if (!$fd) { - return FALSE; - } - - $this->_lineCount = $this->_warningCount = 0; - $this->_invalidRowCount = $this->_validCount = 0; - $this->_totalCount = $this->_conflictCount = 0; - - $this->_errors = []; - $this->_warnings = []; - $this->_conflicts = []; - - $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); - - if ($mode == self::MODE_MAPFIELD) { - $this->_rows = []; - } - else { - $this->_activeFieldCount = count($this->_activeFields); - } - if ($statusID) { - $this->progressImport($statusID); - $startTimestamp = $currTimestamp = $prevTimestamp = time(); - } - - while (!feof($fd)) { - $this->_lineCount++; - - $values = fgetcsv($fd, 8192, $separator); - if (!$values) { - continue; - } - - self::encloseScrub($values); - - // skip column header if we're not in mapfield mode - if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { - $skipColumnHeader = FALSE; - continue; - } - - // Trim whitespace around the values. - - $empty = TRUE; - foreach ($values as $k => $v) { - $values[$k] = trim($v, " \t\r\n"); - } - - if (CRM_Utils_System::isNull($values)) { - continue; - } - - $this->_totalCount++; - - if ($mode == self::MODE_MAPFIELD) { - $returnCode = $this->mapField($values); - } - elseif ($mode == self::MODE_PREVIEW) { - $returnCode = $this->preview($values); - } - elseif ($mode == self::MODE_SUMMARY) { - $returnCode = $this->summary($values); - } - elseif ($mode == self::MODE_IMPORT) { - $returnCode = $this->import($onDuplicate, $values); - if ($statusID && (($this->_lineCount % 50) == 0)) { - $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); - } - } - else { - $returnCode = self::ERROR; - } - - // note that a line could be valid but still produce a warning - if ($returnCode & self::VALID) { - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode & self::WARNING) { - $this->_warningCount++; - if ($this->_warningCount < $this->_maxWarningCount) { - $this->_warningCount[] = $line; - } - } - - if ($returnCode & self::ERROR) { - $this->_invalidRowCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_errors[] = $values; - } - - if ($returnCode & self::CONFLICT) { - $this->_conflictCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_conflicts[] = $values; - } - - if ($returnCode & self::DUPLICATE) { - $this->_duplicateCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_duplicates[] = $values; - if ($onDuplicate != self::DUPLICATE_SKIP) { - $this->_validCount++; - } - } - - // we give the derived class a way of aborting the process - // note that the return code could be multiple code or'ed together - if ($returnCode & self::STOP) { - break; - } - - // if we are done processing the maxNumber of lines, break - if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { - break; - } - } - - fclose($fd); - - if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { - $customHeaders = $mapper; - - $customfields = CRM_Core_BAO_CustomField::getFields('Activity'); - foreach ($customHeaders as $key => $value) { - if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { - $customHeaders[$key] = $customfields[$id][0]; - } - } - if ($this->_invalidRowCount) { - // removed view url for invlaid contacts - $headers = array_merge( - [ts('Line Number'), ts('Reason')], - $customHeaders - ); - $this->_errorFileName = self::errorFileName(self::ERROR); - self::exportCSV($this->_errorFileName, $headers, $this->_errors); - } - if ($this->_conflictCount) { - $headers = array_merge( - [ts('Line Number'), ts('Reason')], - $customHeaders - ); - $this->_conflictFileName = self::errorFileName(self::CONFLICT); - self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts); - } - if ($this->_duplicateCount) { - $headers = array_merge( - [ts('Line Number'), ts('View Activity History URL')], - $customHeaders - ); - - $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); - self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); - } - } - return $this->fini(); - } - - /** - * Given a list of the importable field keys that the user has selected set the active fields array to this list. - * - * @param array $fieldKeys - */ - public function setActiveFields($fieldKeys) { - $this->_activeFieldCount = count($fieldKeys); - foreach ($fieldKeys as $key) { - if (empty($this->_fields[$key])) { - $this->_activeFields[] = new CRM_Activity_Import_Field('', ts('- do not import -')); - } - else { - $this->_activeFields[] = clone($this->_fields[$key]); - } - } - } - - /** - * @param string $name - * @param $title - * @param int $type - * @param string $headerPattern - * @param string $dataPattern - */ - public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') { - if (empty($name)) { - $this->_fields['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern); - } - else { - - $tempField = CRM_Contact_BAO_Contact::importableFields('Individual', NULL); - if (!array_key_exists($name, $tempField)) { - $this->_fields[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern); - } - else { - $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array::value('hasLocationType', $tempField[$name])); - } - } - } - - /** - * Store parser values. - * - * @param CRM_Core_Session $store - * - * @param int $mode - */ - public function set($store, $mode = self::MODE_SUMMARY) { - $store->set('fileSize', $this->_fileSize); - $store->set('lineCount', $this->_lineCount); - $store->set('separator', $this->_separator); - $store->set('fields', $this->getSelectValues()); - - $store->set('headerPatterns', $this->getHeaderPatterns()); - $store->set('dataPatterns', $this->getDataPatterns()); - $store->set('columnCount', $this->_activeFieldCount); - - $store->set('totalRowCount', $this->_totalCount); - $store->set('validRowCount', $this->_validCount); - $store->set('invalidRowCount', $this->_invalidRowCount); - $store->set('conflictRowCount', $this->_conflictCount); - - if ($this->_invalidRowCount) { - $store->set('errorsFileName', $this->_errorFileName); - } - if ($this->_conflictCount) { - $store->set('conflictsFileName', $this->_conflictFileName); - } - if (isset($this->_rows) && !empty($this->_rows)) { - $store->set('dataValues', $this->_rows); - } - - if ($mode == self::MODE_IMPORT) { - $store->set('duplicateRowCount', $this->_duplicateCount); - if ($this->_duplicateCount) { - $store->set('duplicatesFileName', $this->_duplicateFileName); - } - } - } - - /** - * Export data to a CSV file. - * - * @param string $fileName - * @param array $header - * @param array $data - */ - public static function exportCSV($fileName, $header, $data) { - $output = []; - $fd = fopen($fileName, 'w'); - - foreach ($header as $key => $value) { - $header[$key] = "\"$value\""; - } - $config = CRM_Core_Config::singleton(); - $output[] = implode($config->fieldSeparator, $header); - - foreach ($data as $datum) { - foreach ($datum as $key => $value) { - $datum[$key] = "\"$value\""; - } - $output[] = implode($config->fieldSeparator, $datum); - } - fwrite($fd, implode("\n", $output)); - fclose($fd); - } - -} diff --git a/CRM/Activity/Import/Parser/Activity.php b/CRM/Activity/Import/Parser/Activity.php index a1d3cdfa8bd4..7014f313e1f3 100644 --- a/CRM/Activity/Import/Parser/Activity.php +++ b/CRM/Activity/Import/Parser/Activity.php @@ -19,7 +19,7 @@ /** * Class to parse activity csv files. */ -class CRM_Activity_Import_Parser_Activity extends CRM_Activity_Import_Parser { +class CRM_Activity_Import_Parser_Activity extends CRM_Import_Parser { protected $_mapperKeys; @@ -32,6 +32,33 @@ class CRM_Activity_Import_Parser_Activity extends CRM_Activity_Import_Parser { */ protected $_newActivity; + protected $_fileName; + + /** + * Imported file size. + * @var int + */ + protected $_fileSize; + + /** + * Separator being used. + * @var string + */ + protected $_separator; + + /** + * Total number of lines in file. + * @var int + */ + protected $_lineCount; + + /** + * Whether the file has a column header or not. + * + * @var bool + */ + protected $_haveColumnHeader; + /** * Class constructor. * @@ -436,4 +463,319 @@ protected function getApiReadyParams(array $values): array { return $params; } + /** + * @param array $fileName + * @param string $separator + * @param $mapper + * @param bool $skipColumnHeader + * @param int $mode + * @param int $onDuplicate + * @param int $statusID + * @param int $totalRowCount + * + * @return mixed + * @throws Exception + */ + public function run( + array $fileName, + $separator, + $mapper, + $skipColumnHeader = FALSE, + $mode = self::MODE_PREVIEW, + $onDuplicate = self::DUPLICATE_SKIP, + $statusID = NULL, + $totalRowCount = NULL + ) { + + $fileName = $fileName['name']; + + $this->init(); + + $this->_haveColumnHeader = $skipColumnHeader; + + $this->_separator = $separator; + + $fd = fopen($fileName, "r"); + if (!$fd) { + return FALSE; + } + + $this->_lineCount = $this->_warningCount = 0; + $this->_invalidRowCount = $this->_validCount = 0; + $this->_totalCount = $this->_conflictCount = 0; + + $this->_errors = []; + $this->_warnings = []; + $this->_conflicts = []; + + $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); + + if ($mode == self::MODE_MAPFIELD) { + $this->_rows = []; + } + else { + $this->_activeFieldCount = count($this->_activeFields); + } + if ($statusID) { + $this->progressImport($statusID); + $startTimestamp = $currTimestamp = $prevTimestamp = time(); + } + + while (!feof($fd)) { + $this->_lineCount++; + + $values = fgetcsv($fd, 8192, $separator); + if (!$values) { + continue; + } + + self::encloseScrub($values); + + // skip column header if we're not in mapfield mode + if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { + $skipColumnHeader = FALSE; + continue; + } + + // Trim whitespace around the values. + + $empty = TRUE; + foreach ($values as $k => $v) { + $values[$k] = trim($v, " \t\r\n"); + } + + if (CRM_Utils_System::isNull($values)) { + continue; + } + + $this->_totalCount++; + + if ($mode == self::MODE_MAPFIELD) { + $returnCode = $this->mapField($values); + } + elseif ($mode == self::MODE_PREVIEW) { + $returnCode = $this->preview($values); + } + elseif ($mode == self::MODE_SUMMARY) { + $returnCode = $this->summary($values); + } + elseif ($mode == self::MODE_IMPORT) { + $returnCode = $this->import($onDuplicate, $values); + if ($statusID && (($this->_lineCount % 50) == 0)) { + $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); + } + } + else { + $returnCode = self::ERROR; + } + + // note that a line could be valid but still produce a warning + if ($returnCode & self::VALID) { + $this->_validCount++; + if ($mode == self::MODE_MAPFIELD) { + $this->_rows[] = $values; + $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); + } + } + + if ($returnCode & self::WARNING) { + $this->_warningCount++; + if ($this->_warningCount < $this->_maxWarningCount) { + $this->_warningCount[] = $line; + } + } + + if ($returnCode & self::ERROR) { + $this->_invalidRowCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_errors[] = $values; + } + + if ($returnCode & self::CONFLICT) { + $this->_conflictCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_conflicts[] = $values; + } + + if ($returnCode & self::DUPLICATE) { + $this->_duplicateCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_duplicates[] = $values; + if ($onDuplicate != self::DUPLICATE_SKIP) { + $this->_validCount++; + } + } + + // we give the derived class a way of aborting the process + // note that the return code could be multiple code or'ed together + if ($returnCode & self::STOP) { + break; + } + + // if we are done processing the maxNumber of lines, break + if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { + break; + } + } + + fclose($fd); + + if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { + $customHeaders = $mapper; + + $customfields = CRM_Core_BAO_CustomField::getFields('Activity'); + foreach ($customHeaders as $key => $value) { + if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { + $customHeaders[$key] = $customfields[$id][0]; + } + } + if ($this->_invalidRowCount) { + // removed view url for invlaid contacts + $headers = array_merge( + [ts('Line Number'), ts('Reason')], + $customHeaders + ); + $this->_errorFileName = self::errorFileName(self::ERROR); + self::exportCSV($this->_errorFileName, $headers, $this->_errors); + } + if ($this->_conflictCount) { + $headers = array_merge( + [ts('Line Number'), ts('Reason')], + $customHeaders + ); + $this->_conflictFileName = self::errorFileName(self::CONFLICT); + self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts); + } + if ($this->_duplicateCount) { + $headers = array_merge( + [ts('Line Number'), ts('View Activity History URL')], + $customHeaders + ); + + $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); + self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); + } + } + return $this->fini(); + } + + /** + * Given a list of the importable field keys that the user has selected set the active fields array to this list. + * + * @param array $fieldKeys + */ + public function setActiveFields($fieldKeys) { + $this->_activeFieldCount = count($fieldKeys); + foreach ($fieldKeys as $key) { + if (empty($this->_fields[$key])) { + $this->_activeFields[] = new CRM_Activity_Import_Field('', ts('- do not import -')); + } + else { + $this->_activeFields[] = clone($this->_fields[$key]); + } + } + } + + /** + * @param string $name + * @param $title + * @param int $type + * @param string $headerPattern + * @param string $dataPattern + */ + public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') { + if (empty($name)) { + $this->_fields['doNotImport'] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern); + } + else { + + $tempField = CRM_Contact_BAO_Contact::importableFields('Individual', NULL); + if (!array_key_exists($name, $tempField)) { + $this->_fields[$name] = new CRM_Activity_Import_Field($name, $title, $type, $headerPattern, $dataPattern); + } + else { + $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, CRM_Utils_Array::value('hasLocationType', $tempField[$name])); + } + } + } + + /** + * Store parser values. + * + * @param CRM_Core_Session $store + * + * @param int $mode + */ + public function set($store, $mode = self::MODE_SUMMARY) { + $store->set('fileSize', $this->_fileSize); + $store->set('lineCount', $this->_lineCount); + $store->set('separator', $this->_separator); + $store->set('fields', $this->getSelectValues()); + + $store->set('headerPatterns', $this->getHeaderPatterns()); + $store->set('dataPatterns', $this->getDataPatterns()); + $store->set('columnCount', $this->_activeFieldCount); + + $store->set('totalRowCount', $this->_totalCount); + $store->set('validRowCount', $this->_validCount); + $store->set('invalidRowCount', $this->_invalidRowCount); + $store->set('conflictRowCount', $this->_conflictCount); + + if ($this->_invalidRowCount) { + $store->set('errorsFileName', $this->_errorFileName); + } + if ($this->_conflictCount) { + $store->set('conflictsFileName', $this->_conflictFileName); + } + if (isset($this->_rows) && !empty($this->_rows)) { + $store->set('dataValues', $this->_rows); + } + + if ($mode == self::MODE_IMPORT) { + $store->set('duplicateRowCount', $this->_duplicateCount); + if ($this->_duplicateCount) { + $store->set('duplicatesFileName', $this->_duplicateFileName); + } + } + } + + /** + * Export data to a CSV file. + * + * @param string $fileName + * @param array $header + * @param array $data + */ + public static function exportCSV($fileName, $header, $data) { + $output = []; + $fd = fopen($fileName, 'w'); + + foreach ($header as $key => $value) { + $header[$key] = "\"$value\""; + } + $config = CRM_Core_Config::singleton(); + $output[] = implode($config->fieldSeparator, $header); + + foreach ($data as $datum) { + foreach ($datum as $key => $value) { + $datum[$key] = "\"$value\""; + } + $output[] = implode($config->fieldSeparator, $datum); + } + fwrite($fd, implode("\n", $output)); + fclose($fd); + } + }