/*
* Copyright (c) Contributors, http://opensimulator.org/
* See CONTRIBUTORS.TXT for a full list of copyright holders.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of the OpenSimulator Project nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Reflection;
using System.Threading;
using System.Text;
using System.Xml;
using System.Xml.Linq;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
using OpenSim.Framework.Serialization;
using OpenSim.Framework.Serialization.External;
using OpenSim.Region.CoreModules.World.Archiver;
using OpenSim.Region.Framework.Scenes;
using OpenSim.Region.Framework.Scenes.Serialization;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Services.Interfaces;
namespace OpenSim.Region.CoreModules.Avatar.Inventory.Archiver
{
public class InventoryArchiveReadRequest
{
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
///
/// The maximum major version of archive that we can read. Minor versions shouldn't need a max number since version
/// bumps here should be compatible.
///
public static int MAX_MAJOR_VERSION = 1;
protected TarArchiveReader archive;
private UserAccount m_userInfo;
private string m_invPath;
///
/// Do we want to merge this load with existing inventory?
///
protected bool m_merge;
protected IInventoryService m_InventoryService;
protected IAssetService m_AssetService;
protected IUserAccountService m_UserAccountService;
///
/// The stream from which the inventory archive will be loaded.
///
private Stream m_loadStream;
///
/// Has the control file been loaded for this archive?
///
public bool ControlFileLoaded { get; private set; }
///
/// Do we want to enforce the check. IAR versions before 0.2 and 1.1 do not guarantee this order, so we can't
/// enforce.
///
public bool EnforceControlFileCheck { get; private set; }
protected bool m_assetsLoaded;
protected bool m_inventoryNodesLoaded;
protected int m_successfulAssetRestores;
protected int m_failedAssetRestores;
protected int m_successfulItemRestores;
///
/// Root destination folder for the IAR load.
///
protected InventoryFolderBase m_rootDestinationFolder;
///
/// Inventory nodes loaded from the iar.
///
protected HashSet m_loadedNodes = new HashSet();
///
/// In order to load identically named folders, we need to keep track of the folders that we have already
/// resolved.
///
Dictionary m_resolvedFolders = new Dictionary();
///
/// Record the creator id that should be associated with an asset. This is used to adjust asset creator ids
/// after OSP resolution (since OSP creators are only stored in the item
///
protected Dictionary m_creatorIdForAssetId = new Dictionary();
public InventoryArchiveReadRequest(
IInventoryService inv, IAssetService assets, IUserAccountService uacc, UserAccount userInfo, string invPath, string loadPath, bool merge)
: this(
inv,
assets,
uacc,
userInfo,
invPath,
new GZipStream(ArchiveHelpers.GetStream(loadPath), CompressionMode.Decompress),
merge)
{
}
public InventoryArchiveReadRequest(
IInventoryService inv, IAssetService assets, IUserAccountService uacc, UserAccount userInfo, string invPath, Stream loadStream, bool merge)
{
m_InventoryService = inv;
m_AssetService = assets;
m_UserAccountService = uacc;
m_merge = merge;
m_userInfo = userInfo;
m_invPath = invPath;
m_loadStream = loadStream;
// FIXME: Do not perform this check since older versions of OpenSim do save the control file after other things
// (I thought they weren't). We will need to bump the version number and perform this check on all
// subsequent IAR versions only
ControlFileLoaded = true;
}
///
/// Execute the request
///
///
/// Only call this once. To load another IAR, construct another request object.
///
///
/// A list of the inventory nodes loaded. If folders were loaded then only the root folders are
/// returned
///
/// Thrown if load fails.
public HashSet Execute()
{
try
{
string filePath = "ERROR";
List folderCandidates
= InventoryArchiveUtils.FindFoldersByPath(
m_InventoryService, m_userInfo.PrincipalID, m_invPath);
if (folderCandidates.Count == 0)
{
// Possibly provide an option later on to automatically create this folder if it does not exist
m_log.ErrorFormat("[INVENTORY ARCHIVER]: Inventory path {0} does not exist", m_invPath);
return m_loadedNodes;
}
m_rootDestinationFolder = folderCandidates[0];
archive = new TarArchiveReader(m_loadStream);
byte[] data;
TarArchiveReader.TarEntryType entryType;
while ((data = archive.ReadEntry(out filePath, out entryType)) != null)
{
if (filePath == ArchiveConstants.CONTROL_FILE_PATH)
{
LoadControlFile(filePath, data);
}
else if (filePath.StartsWith(ArchiveConstants.ASSETS_PATH))
{
LoadAssetFile(filePath, data);
}
else if (filePath.StartsWith(ArchiveConstants.INVENTORY_PATH))
{
LoadInventoryFile(filePath, entryType, data);
}
}
archive.Close();
m_log.DebugFormat(
"[INVENTORY ARCHIVER]: Successfully loaded {0} assets with {1} failures",
m_successfulAssetRestores, m_failedAssetRestores);
m_log.InfoFormat("[INVENTORY ARCHIVER]: Successfully loaded {0} items", m_successfulItemRestores);
return m_loadedNodes;
}
finally
{
m_loadStream.Close();
}
}
public void Close()
{
if (m_loadStream != null)
m_loadStream.Close();
}
///
/// Replicate the inventory paths in the archive to the user's inventory as necessary.
///
/// The item archive path to replicate
/// The root folder for the inventory load
///
/// The folders that we have resolved so far for a given archive path.
/// This method will add more folders if necessary
///
///
/// Track the inventory nodes created.
///
/// The last user inventory folder created or found for the archive path
public InventoryFolderBase ReplicateArchivePathToUserInventory(
string iarPath,
InventoryFolderBase rootDestFolder,
Dictionary resolvedFolders,
HashSet loadedNodes)
{
string iarPathExisting = iarPath;
// m_log.DebugFormat(
// "[INVENTORY ARCHIVER]: Loading folder {0} {1}", rootDestFolder.Name, rootDestFolder.ID);
InventoryFolderBase destFolder
= ResolveDestinationFolder(rootDestFolder, ref iarPathExisting, resolvedFolders);
// m_log.DebugFormat(
// "[INVENTORY ARCHIVER]: originalArchivePath [{0}], section already loaded [{1}]",
// iarPath, iarPathExisting);
string iarPathToCreate = iarPath.Substring(iarPathExisting.Length);
CreateFoldersForPath(destFolder, iarPathExisting, iarPathToCreate, resolvedFolders, loadedNodes);
return destFolder;
}
///
/// Resolve a destination folder
///
///
/// We require here a root destination folder (usually the root of the user's inventory) and the archive
/// path. We also pass in a list of previously resolved folders in case we've found this one previously.
///
///
/// The item archive path to resolve. The portion of the path passed back is that
/// which corresponds to the resolved desintation folder.
///
/// The root folder for the inventory load
///
///
/// The folders that we have resolved so far for a given archive path.
///
///
/// The folder in the user's inventory that matches best the archive path given. If no such folder was found
/// then the passed in root destination folder is returned.
///
protected InventoryFolderBase ResolveDestinationFolder(
InventoryFolderBase rootDestFolder,
ref string archivePath,
Dictionary resolvedFolders)
{
// string originalArchivePath = archivePath;
while (archivePath.Length > 0)
{
// m_log.DebugFormat("[INVENTORY ARCHIVER]: Trying to resolve destination folder {0}", archivePath);
if (resolvedFolders.ContainsKey(archivePath))
{
// m_log.DebugFormat(
// "[INVENTORY ARCHIVER]: Found previously created folder from archive path {0}", archivePath);
return resolvedFolders[archivePath];
}
else
{
if (m_merge)
{
// TODO: Using m_invPath is totally wrong - what we need to do is strip the uuid from the
// iar name and try to find that instead.
string plainPath = ArchiveConstants.ExtractPlainPathFromIarPath(archivePath);
List folderCandidates
= InventoryArchiveUtils.FindFoldersByPath(
m_InventoryService, m_userInfo.PrincipalID, plainPath);
if (folderCandidates.Count != 0)
{
InventoryFolderBase destFolder = folderCandidates[0];
resolvedFolders[archivePath] = destFolder;
return destFolder;
}
}
// Don't include the last slash so find the penultimate one
int penultimateSlashIndex = archivePath.LastIndexOf("/", archivePath.Length - 2);
if (penultimateSlashIndex >= 0)
{
// Remove the last section of path so that we can see if we've already resolved the parent
archivePath = archivePath.Remove(penultimateSlashIndex + 1);
}
else
{
// m_log.DebugFormat(
// "[INVENTORY ARCHIVER]: Found no previously created folder for archive path {0}",
// originalArchivePath);
archivePath = string.Empty;
return rootDestFolder;
}
}
}
return rootDestFolder;
}
///
/// Create a set of folders for the given path.
///
///
/// The root folder from which the creation will take place.
///
///
/// the part of the iar path that already exists
///
///
/// The path to replicate in the user's inventory from iar
///
///
/// The folders that we have resolved so far for a given archive path.
///
///
/// Track the inventory nodes created.
///
protected void CreateFoldersForPath(
InventoryFolderBase destFolder,
string iarPathExisting,
string iarPathToReplicate,
Dictionary resolvedFolders,
HashSet loadedNodes)
{
string[] rawDirsToCreate = iarPathToReplicate.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < rawDirsToCreate.Length; i++)
{
// m_log.DebugFormat("[INVENTORY ARCHIVER]: Creating folder {0} from IAR", rawDirsToCreate[i]);
if (!rawDirsToCreate[i].Contains(ArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR))
continue;
int identicalNameIdentifierIndex
= rawDirsToCreate[i].LastIndexOf(
ArchiveConstants.INVENTORY_NODE_NAME_COMPONENT_SEPARATOR);
string newFolderName = rawDirsToCreate[i].Remove(identicalNameIdentifierIndex);
newFolderName = InventoryArchiveUtils.UnescapeArchivePath(newFolderName);
UUID newFolderId = UUID.Random();
// Asset type has to be Unknown here rather than Folder, otherwise the created folder can't be
// deleted once the client has relogged.
// The root folder appears to be labelled AssetType.Folder (shows up as "Category" in the client)
// even though there is a AssetType.RootCategory
destFolder
= new InventoryFolderBase(
newFolderId, newFolderName, m_userInfo.PrincipalID,
(short)AssetType.Unknown, destFolder.ID, 1);
m_InventoryService.AddFolder(destFolder);
// Record that we have now created this folder
iarPathExisting += rawDirsToCreate[i] + "/";
m_log.DebugFormat("[INVENTORY ARCHIVER]: Created folder {0} from IAR", iarPathExisting);
resolvedFolders[iarPathExisting] = destFolder;
if (0 == i)
loadedNodes.Add(destFolder);
}
}
///
/// Load an item from the archive
///
/// The archive path for the item
/// The raw item data
/// The root destination folder for loaded items
/// All the inventory nodes (items and folders) loaded so far
protected InventoryItemBase LoadItem(byte[] data, InventoryFolderBase loadFolder)
{
InventoryItemBase item = UserInventoryItemSerializer.Deserialize(data);
// Don't use the item ID that's in the file
item.ID = UUID.Random();
UUID ospResolvedId = OspResolver.ResolveOspa(item.CreatorId, m_UserAccountService);
if (UUID.Zero != ospResolvedId) // The user exists in this grid
{
// m_log.DebugFormat("[INVENTORY ARCHIVER]: Found creator {0} via OSPA resolution", ospResolvedId);
// item.CreatorIdAsUuid = ospResolvedId;
// Don't preserve the OSPA in the creator id (which actually gets persisted to the
// database). Instead, replace with the UUID that we found.
item.CreatorId = ospResolvedId.ToString();
item.CreatorData = string.Empty;
}
else if (item.CreatorData == null || item.CreatorData == String.Empty)
{
item.CreatorId = m_userInfo.PrincipalID.ToString();
// item.CreatorIdAsUuid = new UUID(item.CreatorId);
}
item.Owner = m_userInfo.PrincipalID;
// Reset folder ID to the one in which we want to load it
item.Folder = loadFolder.ID;
// Record the creator id for the item's asset so that we can use it later, if necessary, when the asset
// is loaded.
// FIXME: This relies on the items coming before the assets in the TAR file. Need to create stronger
// checks for this, and maybe even an external tool for creating OARs which enforces this, rather than
// relying on native tar tools.
m_creatorIdForAssetId[item.AssetID] = item.CreatorIdAsUuid;
if (!m_InventoryService.AddItem(item))
m_log.WarnFormat("[INVENTORY ARCHIVER]: Unable to save item {0} in folder {1}", item.Name, item.Folder);
return item;
}
///
/// Load an asset
///
///
///
/// true if asset was successfully loaded, false otherwise
private bool LoadAsset(string assetPath, byte[] data)
{
//IRegionSerialiser serialiser = scene.RequestModuleInterface();
// Right now we're nastily obtaining the UUID from the filename
string filename = assetPath.Remove(0, ArchiveConstants.ASSETS_PATH.Length);
int i = filename.LastIndexOf(ArchiveConstants.ASSET_EXTENSION_SEPARATOR);
if (i == -1)
{
m_log.ErrorFormat(
"[INVENTORY ARCHIVER]: Could not find extension information in asset path {0} since it's missing the separator {1}. Skipping",
assetPath, ArchiveConstants.ASSET_EXTENSION_SEPARATOR);
return false;
}
string extension = filename.Substring(i);
string rawUuid = filename.Remove(filename.Length - extension.Length);
UUID assetId = new UUID(rawUuid);
if (ArchiveConstants.EXTENSION_TO_ASSET_TYPE.ContainsKey(extension))
{
sbyte assetType = ArchiveConstants.EXTENSION_TO_ASSET_TYPE[extension];
if (assetType == (sbyte)AssetType.Unknown)
{
m_log.WarnFormat("[INVENTORY ARCHIVER]: Importing {0} byte asset {1} with unknown type", data.Length, assetId);
}
else if (assetType == (sbyte)AssetType.Object)
{
if (m_creatorIdForAssetId.ContainsKey(assetId))
{
string xmlData = Utils.BytesToString(data);
List sceneObjects = new List();
CoalescedSceneObjects coa = null;
if (CoalescedSceneObjectsSerializer.TryFromXml(xmlData, out coa))
{
// m_log.DebugFormat(
// "[INVENTORY ARCHIVER]: Loaded coalescence {0} has {1} objects", assetId, coa.Count);
if (coa.Objects.Count == 0)
{
m_log.WarnFormat(
"[INVENTORY ARCHIVE READ REQUEST]: Aborting load of coalesced object from asset {0} as it has zero loaded components",
assetId);
return false;
}
sceneObjects.AddRange(coa.Objects);
}
else
{
SceneObjectGroup deserializedObject = SceneObjectSerializer.FromOriginalXmlFormat(xmlData);
if (deserializedObject != null)
{
sceneObjects.Add(deserializedObject);
}
else
{
m_log.WarnFormat(
"[INVENTORY ARCHIVE READ REQUEST]: Aborting load of object from asset {0} as deserialization failed",
assetId);
return false;
}
}
foreach (SceneObjectGroup sog in sceneObjects)
foreach (SceneObjectPart sop in sog.Parts)
if (sop.CreatorData == null || sop.CreatorData == "")
sop.CreatorID = m_creatorIdForAssetId[assetId];
if (coa != null)
data = Utils.StringToBytes(CoalescedSceneObjectsSerializer.ToXml(coa));
else
data = Utils.StringToBytes(SceneObjectSerializer.ToOriginalXmlFormat(sceneObjects[0]));
}
}
//m_log.DebugFormat("[INVENTORY ARCHIVER]: Importing asset {0}, type {1}", uuid, assetType);
AssetBase asset = new AssetBase(assetId, "From IAR", assetType, UUID.Zero.ToString());
asset.Data = data;
m_AssetService.Store(asset);
return true;
}
else
{
m_log.ErrorFormat(
"[INVENTORY ARCHIVER]: Tried to dearchive data with path {0} with an unknown type extension {1}",
assetPath, extension);
return false;
}
}
///
/// Load control file
///
///
///
public void LoadControlFile(string path, byte[] data)
{
XDocument doc = XDocument.Parse(Encoding.ASCII.GetString(data));
XElement archiveElement = doc.Element("archive");
int majorVersion = int.Parse(archiveElement.Attribute("major_version").Value);
int minorVersion = int.Parse(archiveElement.Attribute("minor_version").Value);
string version = string.Format("{0}.{1}", majorVersion, minorVersion);
if (majorVersion > MAX_MAJOR_VERSION)
{
throw new Exception(
string.Format(
"The IAR you are trying to load has major version number of {0} but this version of OpenSim can only load IARs with major version number {1} and below",
majorVersion, MAX_MAJOR_VERSION));
}
ControlFileLoaded = true;
m_log.InfoFormat("[INVENTORY ARCHIVER]: Loading IAR with version {0}", version);
}
///
/// Load inventory file
///
///
///
///
protected void LoadInventoryFile(string path, TarArchiveReader.TarEntryType entryType, byte[] data)
{
if (!ControlFileLoaded)
throw new Exception(
string.Format(
"The IAR you are trying to load does not list {0} before {1}. Aborting load",
ArchiveConstants.CONTROL_FILE_PATH, ArchiveConstants.INVENTORY_PATH));
if (m_assetsLoaded)
throw new Exception(
string.Format(
"The IAR you are trying to load does not list all {0} before {1}. Aborting load",
ArchiveConstants.INVENTORY_PATH, ArchiveConstants.ASSETS_PATH));
path = path.Substring(ArchiveConstants.INVENTORY_PATH.Length);
// Trim off the file portion if we aren't already dealing with a directory path
if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY != entryType)
path = path.Remove(path.LastIndexOf("/") + 1);
InventoryFolderBase foundFolder
= ReplicateArchivePathToUserInventory(
path, m_rootDestinationFolder, m_resolvedFolders, m_loadedNodes);
if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY != entryType)
{
InventoryItemBase item = LoadItem(data, foundFolder);
if (item != null)
{
m_successfulItemRestores++;
// If we aren't loading the folder containing the item then well need to update the
// viewer separately for that item.
if (!m_loadedNodes.Contains(foundFolder))
m_loadedNodes.Add(item);
}
}
m_inventoryNodesLoaded = true;
}
///
/// Load asset file
///
///
///
protected void LoadAssetFile(string path, byte[] data)
{
if (!ControlFileLoaded)
throw new Exception(
string.Format(
"The IAR you are trying to load does not list {0} before {1}. Aborting load",
ArchiveConstants.CONTROL_FILE_PATH, ArchiveConstants.ASSETS_PATH));
if (!m_inventoryNodesLoaded)
throw new Exception(
string.Format(
"The IAR you are trying to load does not list all {0} before {1}. Aborting load",
ArchiveConstants.INVENTORY_PATH, ArchiveConstants.ASSETS_PATH));
if (LoadAsset(path, data))
m_successfulAssetRestores++;
else
m_failedAssetRestores++;
if ((m_successfulAssetRestores) % 50 == 0)
m_log.DebugFormat(
"[INVENTORY ARCHIVER]: Loaded {0} assets...",
m_successfulAssetRestores);
m_assetsLoaded = true;
}
}
}