typo3/sysext/impexp/class.tx_impexp.php

Go to the documentation of this file.

<?php
/***************************************************************
*  Copyright notice
*
*  (c) 1999-2005 Kasper Skaarhoj (kasperYYYY@typo3.com)
*  All rights reserved
*
*  This script is part of the TYPO3 project. The TYPO3 project is
*  free software; you can redistribute it and/or modify
*  it under the terms of the GNU General Public License as published by
*  the Free Software Foundation; either version 2 of the License, or
*  (at your option) any later version.
*
*  The GNU General Public License can be found at
*  http://www.gnu.org/copyleft/gpl.html.
*  A copy is found in the textfile GPL.txt and important notices to the license
*  from the author is found in LICENSE.txt distributed with these scripts.
*
*
*  This script is distributed in the hope that it will be useful,
*  but WITHOUT ANY WARRANTY; without even the implied warranty of
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*  GNU General Public License for more details.
*
*  This copyright notice MUST APPEAR in all copies of the script!
***************************************************************/
require_once(PATH_t3lib.'class.t3lib_tcemain.php');
require_once (PATH_t3lib.'class.t3lib_diff.php');
require_once (PATH_t3lib.'class.t3lib_parsehtml.php');

require_once (PATH_t3lib.'class.t3lib_basicfilefunc.php');
require_once (PATH_t3lib.'class.t3lib_extfilefunc.php');









class tx_impexp {

                // Configuration, general
        var $showStaticRelations = FALSE;               // If set, static relations (not exported) will be shown in overview as well
        var $fileadminFolderName = 'fileadmin'; // Name of the "fileadmin" folder where files for export/import should be located

        var $mode = '';                                                 // Whether "import" or "export" mode of object. Set through init() function
        var $update = FALSE;                                    // Updates all records that has same UID instead of creating new!
        var $doesImport = FALSE;                                // Is set by importData() when an import has been done.

                // Configuration, import
        var $display_import_pid_record = '';            // If set to a page-record, then the preview display of the content will expect this page-record to be the target for the import and accordingly display validation information. This triggers the visual view of the import/export memory to validate if import is possible
        var $suggestedInsertUids = array();             // Used to register the forged UID values for imported records that we want to create with the same UIDs as in the import file. Admin-only feature.
        var $import_mode = array();                             // Setting import modes during update state: as_new, exclude, force_uid
        var $global_ignore_pid = FALSE;                 // If set, PID correct is ignored globally
        var $force_all_UIDS = FALSE;                    // If set, all UID values are forced! (update or import)
        var $showDiff = FALSE;                                  // If set, a diff-view column is added to the overview.
        var $allowPHPScripts = FALSE;                   // If set, and if the user is admin, allow the writing of PHP scripts to fileadmin/ area.
        var $enableLogging = FALSE;                             // Disable logging when importing
        var $softrefInputValues = array();              // Array of values to substitute in editable softreferences.
        var $fileIDMap = array();                               // Mapping between the fileID from import memory and the final filenames they are written to.

                // Configuration, export
        var $maxFileSize = 1000000;             // 1MB max file size
        var $maxRecordSize = 1000000;   // 1MB max record size
        var $maxExportSize = 10000000;  // 10MB max export size
        var $relOnlyTables = array();   // add table names here which are THE ONLY ones which will be included into export if found as relations. '_ALL' will allow all tables.
        var $relStaticTables = array(); // add tables names here which should not be exported with the file. (Where relations should be mapped to same UIDs in target system).
        var $excludeMap = array();              // Exclude map. Keys are table:uid  pairs and if set, records are not added to the export.
        var $softrefCfg = array();              // Soft Reference Token ID modes.
        var $extensionDependencies = array();           // Listing extension dependencies.
        var $dontCompress = 0;                  // Set  by user: If set, compression in t3d files is disabled
        var $includeExtFileResources = 0;       // Boolean, if set, HTML file resources are included.
        var $extFileResourceExtensions = 'html,htm,css';        // Files with external media (HTML/css style references inside)

                // Internal, dynamic:
        var $import_mapId = array();            // After records are written this array is filled with [table][original_uid] = [new_uid]
        var $import_newId = array();            // Keys are [tablename]:[new NEWxxx ids (or when updating it is uids)] while values are arrays with table/uid of the original record it is based on. By the array keys the new ids can be looked up inside tcemain
        var $import_newId_pids = array();       // Page id map for page tree (import)
        var $import_data = array();                     // Internal data accumulation for writing records during import
        var $errorLog = array();                        // Error log.
        var $cache_getRecordPath = array();     // Cache for record paths
        var $checkPID_cache = array();          // Cache of checkPID values.

        var $compress = 0;                                      // Set internally if the gzcompress function exists
        var $dat = array();                                     // Internal import/export memory
        var $fileProcObj = '';                          // File processing object



        /**************************
         *
         * Initialize
         *
         *************************/

        function init($dontCompress=0,$mode='') {
                $this->compress = function_exists('gzcompress');
                $this->dontCompress = $dontCompress;

                $this->mode = $mode;
        }














        /**************************
         *
         * Export / Init + Meta Data
         *
         *************************/

        function setHeaderBasics()      {

                        // Initializing:
                if (is_array($this->softrefCfg))        {
                        foreach($this->softrefCfg as $key => $value)    {
                                if (!strlen($value['mode']))    unset($this->softrefCfg[$key]);
                        }
                }

                        // Setting in header memory:
                $this->dat['header']['XMLversion'] = '1.0';             // Version of file format
                $this->dat['header']['meta'] = array();                 // Initialize meta data array (to put it in top of file)
                $this->dat['header']['relStaticTables'] = $this->relStaticTables;       // Add list of tables to consider static
                $this->dat['header']['excludeMap'] = $this->excludeMap;                         // The list of excluded records
                $this->dat['header']['softrefCfg'] = $this->softrefCfg;                 // Soft Reference mode for elements
                $this->dat['header']['extensionDependencies'] = $this->extensionDependencies;           // List of extensions the import depends on.
        }

        function setCharset($charset)   {
                $this->dat['header']['charset'] = $charset;
        }

        function setMetaData($title,$description,$notes,$packager_username,$packager_name,$packager_email)      {
                $this->dat['header']['meta'] = array(
                        'title' => $title,
                        'description' => $description,
                        'notes' => $notes,
                        'packager_username' => $packager_username,
                        'packager_name' => $packager_name,
                        'packager_email' => $packager_email,
                        'TYPO3_version' => TYPO3_version,
#                       'loaded_extensions' => $GLOBALS['TYPO3_CONF_VARS']['EXT']['extList'],
                        'created' => strftime('%A %e. %B %Y', time())
                );
        }

        function addThumbnail($imgFilepath)     {
                if (@is_file($imgFilepath))     {
                        $imgInfo = @getimagesize($imgFilepath);
                        if (is_array($imgInfo)) {
                                $fileContent = t3lib_div::getUrl($imgFilepath);
                                $this->dat['header']['thumbnail'] = array(
                                        'imgInfo' => $imgInfo,
                                        'content' => $fileContent,
                                        'filesize' => strlen($fileContent),
                                        'filemtime' => filemtime($imgFilepath),
                                        'filename' => basename($imgFilepath)
                                );
                        }
                }
        }











        /**************************
         *
         * Export / Init Page tree
         *
         *************************/

        function setPageTree($idH)      {
                $this->dat['header']['pagetree'] = $this->unsetExcludedSections($idH);
                return $this->flatInversePageTree($this->dat['header']['pagetree']);
        }

        function unsetExcludedSections($idH)    {
                if (is_array($idH))     {
                        reset($idH);
                        while(list($k,$v) = each($idH)) {
                                if ($this->excludeMap['pages:'.$idH[$k]['uid']])        {
                                        unset($idH[$k]);
                                } elseif (is_array($idH[$k]['subrow'])) {
                                        $idH[$k]['subrow'] = $this->unsetExcludedSections($idH[$k]['subrow']);
                                }
                        }
                }
                return $idH;
        }

        function flatInversePageTree($idH,$a=array())   {
                if (is_array($idH))     {
                        $idH = array_reverse($idH);
                        reset($idH);
                        while(list($k,$v) = each($idH)) {
                                $a[$v['uid']] = $v['uid'];
                                if (is_array($v['subrow']))     {
                                        $a = $this->flatInversePageTree($v['subrow'],$a);
                                }
                        }
                }
                return $a;
        }

        function flatInversePageTree_pid($idH,$a=array(),$pid=-1)       {
                if (is_array($idH))     {
                        $idH = array_reverse($idH);
                        reset($idH);
                        while(list($k,$v) = each($idH)) {
                                $a[$v['uid']] = $pid;
                                if (is_array($v['subrow']))     {
                                        $a = $this->flatInversePageTree_pid($v['subrow'],$a,$v['uid']);
                                }
                        }
                }
                return $a;
        }











        /**************************
         *
         * Export
         *
         *************************/

        function export_addRecord($table,$row,$relationLevel=0) {
                if (strcmp($table,'') && is_array($row) && $row['uid']>0 && !$this->excludeMap[$table.':'.$row['uid']]) {
                        if ($this->checkPID($table==='pages' ? $row['uid'] : $row['pid']))      {
                                if (!isset($this->dat['records'][$table.':'.$row['uid']]))      {

                                                // Prepare header info:
                                        $headerInfo = array();
                                        $headerInfo['uid'] = $row['uid'];
                                        $headerInfo['pid'] = $row['pid'];
                                        $headerInfo['title'] = t3lib_div::fixed_lgd_cs(t3lib_BEfunc::getRecordTitle($table,$row),40);
                                        $headerInfo['size'] = strlen(serialize($row));
                                        if ($relationLevel)     {
                                                $headerInfo['relationLevel'] = $relationLevel;
                                        }

                                                // If record content is not too large in size, set the header content and add the rest:
                                        if ($headerInfo['size']<$this->maxRecordSize)   {

                                                        // Set the header summary:
                                                $this->dat['header']['records'][$table][$row['uid']] = $headerInfo;

                                                        // Create entry in the PID lookup:
                                                $this->dat['header']['pid_lookup'][$row['pid']][$table][$row['uid']]=1;

                                                        // Data:
                                                $this->dat['records'][$table.':'.$row['uid']] = array();
                                                $this->dat['records'][$table.':'.$row['uid']]['data'] = $row;
                                                $this->dat['records'][$table.':'.$row['uid']]['rels'] = $this->getRelations($table,$row);

                                                        // Add information about the relations in the record in the header:
                                                $this->dat['header']['records'][$table][$row['uid']]['rels'] = $this->flatDBrels($this->dat['records'][$table.':'.$row['uid']]['rels']);

                                                        // Add information about the softrefs to header:
                                                $this->dat['header']['records'][$table][$row['uid']]['softrefs'] = $this->flatSoftRefs($this->dat['records'][$table.':'.$row['uid']]['rels']);

                                        } else $this->error('Record '.$table.':'.$row['uid'].' was larger than maxRecordSize ('.t3lib_div::formatSize($this->maxRecordSize).')');
                                } else $this->error('Record '.$table.':'.$row['uid'].' already added.');
                        } else $this->error('Record '.$table.':'.$row['uid'].' was outside your DB mounts!');
                }
        }

        function export_addDBRelations($relationLevel=0)        {
                global $TCA;

                        // Initialize:
                $addR = array();

                        // Traverse all "rels" registered for "records"
                if (is_array($this->dat['records']))    {
                        reset($this->dat['records']);
                        while(list($k) = each($this->dat['records']))   {
                                if (is_array($this->dat['records'][$k]))        {
                                        reset($this->dat['records'][$k]['rels']);
                                        while(list($fieldname,$vR) = each($this->dat['records'][$k]['rels']))   {
#debug($vR);
                                                        // For all DB types of relations:
                                                if ($vR['type']=='db')  {
                                                        foreach($vR['itemArray'] as $fI)        {
                                                                $this->export_addDBRelations_registerRelation($fI, $addR);
                                                        }
                                                }

                                                        // For all flex/db types of relations:
                                                if ($vR['type']=='flex')        {
                                                                // DB relations in flex form fields:
                                                        if (is_array($vR['flexFormRels']['db']))        {
                                                                foreach($vR['flexFormRels']['db'] as $subList)  {
                                                                        foreach($subList as $fI)        {
                                                                                $this->export_addDBRelations_registerRelation($fI, $addR);
                                                                        }
                                                                }
                                                        }
                                                                // DB oriented soft references in flex form fields:
                                                        if (is_array($vR['flexFormRels']['softrefs']))  {
                                                                foreach($vR['flexFormRels']['softrefs'] as $subList)    {
                                                                        foreach($subList['keys'] as $spKey => $elements)        {
                                                                                foreach($elements as $el)       {
                                                                                        if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID']))    {
                                                                                                list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
                                                                                                $fI = array(
                                                                                                        'table' => $tempTable,
                                                                                                        'id' => $tempUid
                                                                                                );
                                                                                                $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
                                                                                        }
                                                                                }
                                                                        }
                                                                }
                                                        }
                                                }

                                                        // In any case, if there are soft refs:
                                                if (is_array($vR['softrefs']['keys']))  {
                                                        foreach($vR['softrefs']['keys'] as $spKey => $elements) {
                                                                foreach($elements as $el)       {
                                                                        if ($el['subst']['type'] === 'db' && $this->includeSoftref($el['subst']['tokenID']))    {
                                                                                list($tempTable, $tempUid) = explode(':', $el['subst']['recordRef']);
                                                                                $fI = array(
                                                                                        'table' => $tempTable,
                                                                                        'id' => $tempUid
                                                                                );
                                                                                $this->export_addDBRelations_registerRelation($fI, $addR, $el['subst']['tokenID']);
                                                                        }
                                                                }
                                                        }
                                                }
                                        }
                                }
                        }
                } else $this->error('There were no records available.');

                        // Now, if there were new records to add, do so:
                if (count($addR))       {
                        foreach($addR as $fI)   {

                                        // Get and set record:
                                $row = t3lib_BEfunc::getRecord($fI['table'],$fI['id']);
                                if (is_array($row))     {
                                        $this->export_addRecord($fI['table'],$row,$relationLevel+1);
                                }

                                        // Set status message
                                if ($fI['id']>0)        {       // Relation pointers always larger than zero except certain "select" types with negative values pointing to uids - but that is not supported here.
                                        $rId = $fI['table'].':'.$fI['id'];
                                        if (!isset($this->dat['records'][$rId]))        {
                                                $this->dat['records'][$rId] = 'NOT_FOUND';
                                                $this->error('Relation record '.$rId.' was not found!');
                                        }
                                }
                        }
                }

                        // Return overview of relations found and added
                return $addR;
        }

        function export_addDBRelations_registerRelation($fI, &$addR, $tokenID='')       {
                global $TCA;

                $rId = $fI['table'].':'.$fI['id'];
                if (isset($TCA[$fI['table']])
                                && !$this->isTableStatic($fI['table'])
                                && !$this->isExcluded($fI['table'],$fI['id'])
                                && (!$tokenID || $this->includeSoftref($tokenID))
                                && $this->inclRelation($fI['table'])
                                )       {
                        if (!isset($this->dat['records'][$rId]))        {
                                        // Set this record to be included since it is not already.
                                $addR[$rId] = $fI;
                        }
                }
        }

        function export_addFilesFromRelations() {

                        // Traverse all "rels" registered for "records"
                if (is_array($this->dat['records']))    {
                        reset($this->dat['records']);
                        while(list($k)=each($this->dat['records']))     {
                                if (is_array($this->dat['records'][$k]['rels']))        {
                                        reset($this->dat['records'][$k]['rels']);
                                        while(list($fieldname,$vR)=each($this->dat['records'][$k]['rels']))     {

                                                        // For all file type relations:
                                                if ($vR['type']=='file')        {
                                                        foreach($vR['newValueFiles'] as $key => $fI)    {
                                                                $this->export_addFile($fI, $k, $fieldname);
                                                                        // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
                                                                unset($this->dat['records'][$k]['rels'][$fieldname]['newValueFiles'][$key]['ID_absFile']);
                                                        }
                                                }

                                                        // For all flex type relations:
                                                if ($vR['type']=='flex')        {
                                                        if (is_array($vR['flexFormRels']['file']))      {
                                                                foreach($vR['flexFormRels']['file'] as $key => $subList)        {
                                                                        foreach($subList as $subKey => $fI)     {
                                                                                $this->export_addFile($fI, $k, $fieldname);
                                                                                        // Remove the absolute reference to the file so it doesn't expose absolute paths from source server:
                                                                                unset($this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['file'][$key][$subKey]['ID_absFile']);
                                                                        }
                                                                }
                                                        }

                                                                // DB oriented soft references in flex form fields:
                                                        if (is_array($vR['flexFormRels']['softrefs']))  {
                                                                foreach($vR['flexFormRels']['softrefs'] as $key => $subList)    {
                                                                        foreach($subList['keys'] as $spKey => $elements)        {
                                                                                foreach($elements as $subKey => $el)    {
                                                                                        if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID']))  {

                                                                                                        // Create abs path and ID for file:
                                                                                                $ID_absFile = t3lib_div::getFileAbsFileName(PATH_site.$el['subst']['relFileName']);
                                                                                                $ID = md5($ID_absFile);

                                                                                                if ($ID_absFile)        {
                                                                                                        if (!$this->dat['files'][$ID])  {
                                                                                                                $fI = array(
                                                                                                                        'filename' => basename($ID_absFile),
                                                                                                                        'ID_absFile' => $ID_absFile,
                                                                                                                        'ID' => $ID,
                                                                                                                        'relFileName' => $el['subst']['relFileName']
                                                                                                                );
                                                                                                                $this->export_addFile($fI, '_SOFTREF_');
                                                                                                        }
                                                                                                        $this->dat['records'][$k]['rels'][$fieldname]['flexFormRels']['softrefs'][$key]['keys'][$spKey][$subKey]['file_ID'] = $ID;
                                                                                                }
                                                                                        }
                                                                                }
                                                                        }
                                                                }
                                                        }
                                                }

                                                        // In any case, if there are soft refs:
                                                if (is_array($vR['softrefs']['keys']))  {
                                                        foreach($vR['softrefs']['keys'] as $spKey => $elements) {
                                                                foreach($elements as $subKey => $el)    {
                                                                        if ($el['subst']['type'] === 'file' && $this->includeSoftref($el['subst']['tokenID']))  {

                                                                                        // Create abs path and ID for file:
                                                                                $ID_absFile = t3lib_div::getFileAbsFileName(PATH_site.$el['subst']['relFileName']);
                                                                                $ID = md5($ID_absFile);

                                                                                if ($ID_absFile)        {
                                                                                        if (!$this->dat['files'][$ID])  {
                                                                                                $fI = array(
                                                                                                        'filename' => basename($ID_absFile),
                                                                                                        'ID_absFile' => $ID_absFile,
                                                                                                        'ID' => $ID,
                                                                                                        'relFileName' => $el['subst']['relFileName']
                                                                                                );
                                                                                                $this->export_addFile($fI, '_SOFTREF_');
                                                                                        }
                                                                                        $this->dat['records'][$k]['rels'][$fieldname]['softrefs']['keys'][$spKey][$subKey]['file_ID'] = $ID;
                                                                                }
                                                                        }
                                                                }
                                                        }
                                                }
                                        }
                                }
                        }
                } else $this->error('There were no records available.');
        }

        function export_addFile($fI, $recordRef='', $fieldname='')      {
                if (@is_file($fI['ID_absFile']))        {
                        if (filesize($fI['ID_absFile']) < $this->maxFileSize)   {
                                $fileRec = array();
                                $fileRec['filesize'] = filesize($fI['ID_absFile']);
                                $fileRec['filename'] = basename($fI['ID_absFile']);
                                $fileRec['filemtime'] = filemtime($fI['ID_absFile']);
                                if ($recordRef) {
                                        $fileRec['record_ref'] = $recordRef.'/'.$fieldname;
                                }
                                if ($fI['relFileName']) {
                                        $fileRec['relFileName'] = $fI['relFileName'];
                                }

                                        // Setting this data in the header
                                $this->dat['header']['files'][$fI['ID']] = $fileRec;

                                        // ... and for the recordlisting, why not let us know WHICH relations there was...
                                if ($recordRef && $recordRef!=='_SOFTREF_')     {
                                        $refParts = explode(':',$recordRef,2);
                                        if (!is_array($this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'])) {
                                                $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'] = array();
                                        }
                                        $this->dat['header']['records'][$refParts[0]][$refParts[1]]['filerefs'][] = $fI['ID'];
                                }

                                        // ... and finally add the heavy stuff:
                                $fileRec['content'] = t3lib_div::getUrl($fI['ID_absFile']);
                                $fileRec['content_md5'] = md5($fileRec['content']);
                                $this->dat['files'][$fI['ID']] = $fileRec;


                                        // For soft references, do further processing:
                                if ($recordRef === '_SOFTREF_') {

                                                // RTE files?
                                        if ($RTEoriginal = $this->getRTEoriginalFilename(basename($fI['ID_absFile'])))  {
                                                $RTEoriginal_absPath = dirname($fI['ID_absFile']).'/'.$RTEoriginal;
                                                if (@is_file($RTEoriginal_absPath))     {

                                                        $RTEoriginal_ID = md5($RTEoriginal_absPath);

                                                        $fileRec = array();
                                                        $fileRec['filesize'] = filesize($RTEoriginal_absPath);
                                                        $fileRec['filename'] = basename($RTEoriginal_absPath);
                                                        $fileRec['filemtime'] = filemtime($RTEoriginal_absPath);
                                                        $fileRec['record_ref'] = '_RTE_COPY_ID:'.$fI['ID'];
                                                        $this->dat['header']['files'][$fI['ID']]['RTE_ORIG_ID'] = $RTEoriginal_ID;

                                                                // Setting this data in the header
                                                        $this->dat['header']['files'][$RTEoriginal_ID] = $fileRec;

                                                                // ... and finally add the heavy stuff:
                                                        $fileRec['content'] = t3lib_div::getUrl($RTEoriginal_absPath);
                                                        $fileRec['content_md5'] = md5($fileRec['content']);
                                                        $this->dat['files'][$RTEoriginal_ID] = $fileRec;
                                                } else {
                                                        $this->error('RTE original file "'.substr($RTEoriginal_absPath,strlen(PATH_site)).'" was not found!');
                                                }
                                        }

                                                // Files with external media?
                                                // This is only done with files grabbed by a softreference parser since it is deemed improbable that hard-referenced files should undergo this treatment.
                                        $html_fI = pathinfo(basename($fI['ID_absFile']));
                                        if ($this->includeExtFileResources && t3lib_div::inList($this->extFileResourceExtensions,strtolower($html_fI['extension'])))    {
                                                $uniquePrefix = '###'.md5(time()).'###';

                                                if (strtolower($html_fI['extension'])==='css')  {
                                                        $prefixedMedias = explode($uniquePrefix, eregi_replace('(url[[:space:]]*\([[:space:]]*["\']?)([^"\')]*)(["\']?[[:space:]]*\))', '\1'.$uniquePrefix.'\2'.$uniquePrefix.'\3', $fileRec['content']));
                                                } else {        // html, htm:
                                                        $htmlParser = t3lib_div::makeInstance('t3lib_parsehtml');
                                                        $prefixedMedias = explode($uniquePrefix, $htmlParser->prefixResourcePath($uniquePrefix,$fileRec['content'],array(),$uniquePrefix));
                                                }

                                                $htmlResourceCaptured = FALSE;
                                                foreach($prefixedMedias as $k => $v)    {
                                                        if ($k%2)       {
                                                                $EXTres_absPath = t3lib_div::resolveBackPath(dirname($fI['ID_absFile']).'/'.$v);
                                                                $EXTres_absPath = t3lib_div::getFileAbsFileName($EXTres_absPath);
                                                                if ($EXTres_absPath && t3lib_div::isFirstPartOfStr($EXTres_absPath,PATH_site.$this->fileadminFolderName.'/') && @is_file($EXTres_absPath))      {

                                                                        $htmlResourceCaptured = TRUE;
                                                                        $EXTres_ID = md5($EXTres_absPath);
                                                                        $this->dat['header']['files'][$fI['ID']]['EXT_RES_ID'][] = $EXTres_ID;
                                                                        $prefixedMedias[$k] = '{EXT_RES_ID:'.$EXTres_ID.'}';

                                                                                // Add file to memory if it is not set already:
                                                                        if (!isset($this->dat['header']['files'][$EXTres_ID]))          {
                                                                                $fileRec = array();
                                                                                $fileRec['filesize'] = filesize($EXTres_absPath);
                                                                                $fileRec['filename'] = basename($EXTres_absPath);
                                                                                $fileRec['filemtime'] = filemtime($EXTres_absPath);
                                                                                $fileRec['record_ref'] = '_EXT_PARENT_:'.$fI['ID'];

                                                                                $fileRec['parentRelFileName'] = $v;             // Media relative to the HTML file.

                                                                                        // Setting this data in the header
                                                                                $this->dat['header']['files'][$EXTres_ID] = $fileRec;

                                                                                        // ... and finally add the heavy stuff:
                                                                                $fileRec['content'] = t3lib_div::getUrl($EXTres_absPath);
                                                                                $fileRec['content_md5'] = md5($fileRec['content']);
                                                                                $this->dat['files'][$EXTres_ID] = $fileRec;
                                                                        }
                                                                }
                                                        }
                                                }

                                                if ($htmlResourceCaptured)      {
                                                        $this->dat['files'][$fI['ID']]['tokenizedContent'] = implode('', $prefixedMedias);
                                                }
                                        }
                                }

                        } else  $this->error($fI['ID_absFile'].' was larger ('.t3lib_div::formatSize(filesize($fI['ID_absFile'])).') than the maxFileSize ('.t3lib_div::formatSize($this->maxFileSize).')! Skipping.');
                } else $this->error($fI['ID_absFile'].' was not a file! Skipping.');
        }

        function getRelations($table,$row)      {
                global $TCA;

                        // Load full table description
                t3lib_div::loadTCA($table);

                        // Initialize:
                $uid = $row['uid'];
                $nonFields = explode(',','uid,perms_userid,perms_groupid,perms_user,perms_group,perms_everybody,pid');

                $outRow = array();
                foreach($row as $field => $value)       {
                        if (!in_array($field,$nonFields) && is_array($TCA[$table]['columns'][$field]))  {
                                $conf = $TCA[$table]['columns'][$field]['config'];

                                        // Add files
                                if ($result = $this->getRelations_procFiles($value, $conf, $uid))       {
                                                // Creates an entry for the field with all the files:
                                        $outRow[$field] = array(
                                                'type' => 'file',
                                                'newValueFiles' => $result,
                                        );
                                }

                                        // Add DB:
                                if ($result = $this->getRelations_procDB($value, $conf, $uid))  {
                                                // Create an entry for the field with all DB relations:
                                        $outRow[$field] = array(
                                                'type' => 'db',
                                                'itemArray' => $result,
                                        );
                                }

                                        // For "flex" fieldtypes we need to traverse the structure looking for file and db references of course!
                                if ($conf['type']=='flex')      {

                                                // Get current value array:
                                        $dataStructArray = t3lib_BEfunc::getFlexFormDS($conf, $row, $table);
                                        $currentValueArray = t3lib_div::xml2array($value);

                                                // Traversing the XML structure, processing files:
                                        if (is_array($currentValueArray))       {
                                                $this->temp_flexRelations = array(
                                                        'db' => array(),
                                                        'file' => array(),
                                                        'softrefs' => array()
                                                );

                                                $iteratorObj = t3lib_div::makeInstance('t3lib_TCEmain');
                                                $iteratorObj->callBackObj = &$this;
                                                $iteratorObj->checkValue_flex_procInData(
                                                                        $currentValueArray['data'],
                                                                        array(),        // Not used.
                                                                        array(),        // Not used.
                                                                        $dataStructArray,
                                                                        array($table,$uid,$field),      // Parameters.
                                                                        'getRelations_flexFormCallBack'
                                                                );

                                                        // Create an entry for the field:
                                                $outRow[$field] = array(
                                                        'type' => 'flex',
                                                        'flexFormRels' => $this->temp_flexRelations,
                                                );
#debug($outRow[$field]);
                                        }
                                }

                                        // Soft References:
                                if (strlen($value) && $softRefs = t3lib_BEfunc::explodeSoftRefParserList($conf['softref'], $table, $field))     {
                                        $softRefValue = $value;
                                        foreach($softRefs as $spKey => $spParams)       {
                                                $softRefObj = &t3lib_BEfunc::softRefParserObj($spKey);
                                                if (is_object($softRefObj))     {
                                                        $resultArray = $softRefObj->findRef($table, $field, $uid, $softRefValue, $spKey, $spParams);
                                                        if (is_array($resultArray))     {
                                                                $outRow[$field]['softrefs']['keys'][$spKey] = $resultArray['elements'];
                                                                if (strlen($resultArray['content'])) {
                                                                        $softRefValue = $resultArray['content'];
                                                                }
                                                        }
                                                }
                                        }

                                        if (is_array($outRow[$field]['softrefs']) && count($outRow[$field]['softrefs']) && strcmp($value,$softRefValue) && strstr($softRefValue,'{softref:'))   {
#debug($softRefValue,'tokenizedContent');
                                                $outRow[$field]['softrefs']['tokenizedContent'] = $softRefValue;
                                        }
                                }
                        }
                }

                return $outRow;
        }

        function getRelations_flexFormCallBack($pParams, $dsConf, $dataValue, $dataValue_ext1, $dataValue_ext2, $structurePath) {

                        // Implode parameter values:
                list($table, $uid, $field) = $pParams;

                        // Add files
                if ($result = $this->getRelations_procFiles($dataValue, $dsConf, $uid)) {

                                // Creates an entry for the field with all the files:
                        $this->temp_flexRelations['file'][$structurePath] = $result;
                }

                        // Add DB:
                if ($result = $this->getRelations_procDB($dataValue, $dsConf, $uid))    {

                                // Create an entry for the field with all DB relations:
                        $this->temp_flexRelations['db'][$structurePath] = $result;
                }

                        // Soft References:
                if (strlen($dataValue) && $softRefs = t3lib_BEfunc::explodeSoftRefParserList($dsConf['softref'], $table, $field))       {
#debug($softRefs);
                        $softRefValue = $dataValue;
                        foreach($softRefs as $spKey => $spParams)       {
                                $softRefObj = &t3lib_BEfunc::softRefParserObj($spKey);
                                if (is_object($softRefObj))     {
                                        $resultArray = $softRefObj->findRef($table, $field, $uid, $softRefValue, $spKey, $spParams, $structurePath);
                                        if (is_array($resultArray) && is_array($resultArray['elements']))       {
                                                $this->temp_flexRelations['softrefs'][$structurePath]['keys'][$spKey] = $resultArray['elements'];
                                                if (strlen($resultArray['content'])) $softRefValue = $resultArray['content'];
                                        }
                                }
                        }

                        if (count($this->temp_flexRelations['softrefs']) && strcmp($dataValue,$softRefValue))   {
                                $this->temp_flexRelations['softrefs'][$structurePath]['tokenizedContent'] = $softRefValue;
#debug($this->temp_flexRelations['softrefs'][$structurePath], $structurePath);
                        }
                }
        }

        function getRelations_procFiles($value, $conf, $uid)    {
                        // Take care of files...
                if ($conf['type']=='group' && $conf['internal_type']=='file')   {

                                // Collect file values in array:
                        if ($conf['MM'])        {
                                $theFileValues = array();
                                $dbAnalysis = t3lib_div::makeInstance('t3lib_loadDBGroup');
                                $dbAnalysis->start('', 'files', $conf['MM'], $uid);

                                foreach($dbAnalysis->itemArray as $somekey => $someval) {
                                        if ($someval['id'])     {
                                                $theFileValues[] = $someval['id'];
                                        }
                                }
                        } else {
                                $theFileValues = explode(',',$value);
                        }

                                // Traverse the files and add them:
                        $uploadFolder = $conf['uploadfolder'];
                        $dest = $this->destPathFromUploadFolder($uploadFolder);
                        $newValue = array();
                        $newValueFiles = array();

                        foreach($theFileValues as $file)        {
                                if (trim($file))        {
                                        $realFile = $dest.'/'.trim($file);
                                        if (@is_file($realFile))        {
                                                $newValueFiles[] = array(
                                                        'filename' => $file,
                                                        'ID' => md5($realFile),
                                                        'ID_absFile' => $realFile
                                                );      // the order should be preserved here because.. (?)
                                        } else $this->error('Missing file: '.$realFile);
                                }
                        }

                        return $newValueFiles;
                }
        }

        function getRelations_procDB($value, $conf, $uid)       {

                        // DB record lists:
                if ($this->isReferenceField($conf))     {
                        $allowedTables = $conf['type']=='group' ? $conf['allowed'] : $conf['foreign_table'].','.$conf['neg_foreign_table'];
                        $prependName = $conf['type']=='group' ? $conf['prepend_tname'] : $conf['neg_foreign_table'];

                        $dbAnalysis = t3lib_div::makeInstance('t3lib_loadDBGroup');
                        $dbAnalysis->start($value,$allowedTables,$conf['MM'],$uid);

                        return $dbAnalysis->itemArray;
                }
        }

        function flatDBrels($dbrels)    {
                $list = array();

                foreach($dbrels as $dat)        {
                        if ($dat['type']=='db') {
                                foreach($dat['itemArray'] as $i)        {
                                        $list[$i['table'].':'.$i['id']] = $i;
                                }
                        }
                        if ($dat['type']=='flex' && is_array($dat['flexFormRels']['db']))       {
                                foreach($dat['flexFormRels']['db'] as $subList) {
                                        foreach($subList as $i) {
                                                $list[$i['table'].':'.$i['id']] = $i;
                                        }
                                }
                        }
                }
                return $list;
        }

        function flatSoftRefs($dbrels)  {
                $list = array();
#debug($dbrels);
                foreach($dbrels as $field => $dat)      {
                        if (is_array($dat['softrefs']['keys'])) {
                                foreach($dat['softrefs']['keys'] as $spKey => $elements)        {
                                        if (is_array($elements))        {
                                                foreach($elements as $subKey => $el)    {
                                                        $lKey = $field.':'.$spKey.':'.$subKey;
                                                        $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey),$el);

                                                                // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
                                                        if ($el['subst'] && $el['subst']['relFileName'])        {
                                                                $list[$lKey]['file_ID'] = md5(PATH_site.$el['subst']['relFileName']);
                                                        }
                                                }
                                        }
                                }
                        }
                        if ($dat['type']=='flex' && is_array($dat['flexFormRels']['softrefs'])) {
                                foreach($dat['flexFormRels']['softrefs'] as $structurePath => $subSoftrefs)     {
                                        if (is_array($subSoftrefs['keys']))     {
                                                foreach($subSoftrefs['keys'] as $spKey => $elements)    {
                                                        foreach($elements as $subKey => $el)    {
                                                                $lKey = $field.':'.$structurePath.':'.$spKey.':'.$subKey;
                                                                $list[$lKey] = array_merge(array('field' => $field, 'spKey' => $spKey, 'structurePath' => $structurePath),$el);

                                                                        // Add file_ID key to header - slightly "risky" way of doing this because if the calculation changes for the same value in $this->records[...] this will not work anymore!
                                                                if ($el['subst'] && $el['subst']['relFileName'])        {
                                                                        $list[$lKey]['file_ID'] = md5(PATH_site.$el['subst']['relFileName']);
                                                                }
                                                        }
                                                }
                                        }
                                }
                        }
                }

#debug($list);
                return $list;
        }

        function destPathFromUploadFolder($folder)      {
                return PATH_site.$folder;
        }










        /**************************
         *
         * File Output
         *
         *************************/

        function compileMemoryToFileContent($type='')   {

                if ($type=='xml')       {
                        $out = $this->createXML();
                } else {
                        $compress = $this->doOutputCompress();
                        $out = '';

                        // adding header:
                        $out.= $this->addFilePart(serialize($this->dat['header']),$compress);

                        // adding records:
                        $out.= $this->addFilePart(serialize($this->dat['records']),$compress);

                        // adding files:
                        $out.= $this->addFilePart(serialize($this->dat['files']),$compress);
                }

                return $out;
        }

        function createXML()    {

                        // Options:
                $options = array(
                        'alt_options' => array(
                                '/header' => array(
                                        'disableTypeAttrib' => TRUE,
                                        'clearStackPath' => TRUE,
                                        'parentTagMap' => array(
                                                'files' => 'file',
                                                'records' => 'table',
                                                'table' => 'rec',
                                                'rec:rels' => 'relations',
                                                'relations' => 'element',
                                                'filerefs' => 'file',
                                                'pid_lookup' => 'page_contents',
                                                'header:relStaticTables' => 'static_tables',
                                                'static_tables' => 'tablename',
                                                'excludeMap' => 'item',
                                                'softrefCfg' => 'softrefExportMode',
                                                'extensionDependencies' => 'extkey',
                                                'softrefs' => 'softref_element',
                                        ),
                                        'alt_options' => array(
                                                '/pagetree' => array(
                                                        'disableTypeAttrib' => TRUE,
                                                        'useIndexTagForNum' => 'node',
                                                        'parentTagMap' => array(
                                                                'node:subrow' => 'node'
                                                        )
                                                ),
                                                '/pid_lookup/page_contents' => array(
                                                        'disableTypeAttrib' => TRUE,
                                                        'parentTagMap' => array(
                                                                'page_contents' => 'table'
                                                        ),
                                                        'grandParentTagMap' => array(
                                                                'page_contents/table' => 'item'
                                                        )
                                                )
                                        )
                                ),
                                '/records' => array(
                                        'disableTypeAttrib' => TRUE,
                                        'parentTagMap' => array(
                                                'records' => 'tablerow',
                                                'tablerow:data' => 'fieldlist',
                                                'tablerow:rels' => 'related',
                                                'related' => 'field',
                                                'field:itemArray' => 'relations',
                                                'field:newValueFiles' => 'filerefs',
                                                'field:flexFormRels' => 'flexform',
                                                'relations' => 'element',
                                                'filerefs' => 'file',
                                                'flexform:db' => 'db_relations',
                                                'flexform:file' => 'file_relations',
                                                'flexform:softrefs' => 'softref_relations',
                                                'softref_relations' => 'structurePath',
                                                'db_relations' => 'path',
                                                'file_relations' => 'path',
                                                'path' => 'element',
                                                'keys' => 'softref_key',
                                                'softref_key' => 'softref_element',
                                        ),
                                        'alt_options' => array(
                                                '/records/tablerow/fieldlist' => array(
                                                        'useIndexTagForAssoc' => 'field',
                                                )
                                        )
                                ),
                                '/files' => array(
                                        'disableTypeAttrib' => TRUE,
                                        'parentTagMap' => array(
                                                'files' => 'file',
                                        ),
                                ),
                        )
                );

                        // Creating XML file from $outputArray:
                $charset = $this->dat['header']['charset'] ? $this->dat['header']['charset'] : 'iso-8859-1';
                $XML = '<?xml version="1.0" encoding="'.$charset.'" standalone="yes" ?>'.chr(10);
                $XML.= t3lib_div::array2xml($this->dat,'',0,'T3RecordDocument',0,$options);

                return $XML;
        }

        function doOutputCompress()     {
                return $this->compress && !$this->dontCompress;
        }

        function addFilePart($data, $compress=FALSE)    {
                if ($compress)  $data = gzcompress($data);
                return md5($data).':'.
                                ($compress?'1':'0').':'.
                                str_pad(strlen($data),10,'0',STR_PAD_LEFT).':'.
                                $data.':';
        }