improve performances

@xen-orchestra/backups/disks/HashedDisk.mjs

@@ -0,0 +1,140 @@
+import { RemoteDisk } from "./RemoteDisk.mjs";
+
+
+/**
+ * this class stores disk blocks by there has instead of their position
+ * doing per disk dedup as a bonus
+ * 
+ * it also integrate the alias part to simplify the rename 
+ * 
+ * determine the extension of these new disk ( Vates Backup Disk ? Hashed Backup Storage ?)
+ * 
+ * {diskname}.{extension}
+ *   |- blockStorePath
+ *          |- hash1
+ *              |- hash2
+ *                  ...
+ *                  |- hashn 
+ */
+
+/**
+ * @typedef {Object} HashedDiskMetadata
+ * @property {number} virtualSize 
+ * @property {string} uuid 
+ * @property {string|undefined} parentUuid 
+ * @property {string|undefined} parentPath 
+ * @property {number} blockSize 
+ * @property {PER_DISK | PER_BACKUP_REPOSITORY  } dedupType  implement PER_DISK here, PER_BACKUP_REPOSITORY will be for later
+ * @property {string} blockStorePath path relative to this file, random name 
+ */
+
+export class HashedDisk extends RemoteDisk{
+   
+  async canMergeConcurently() {
+    return true
+  }
+ 
+  /**
+   * Checks if the hash is not empty in the BAT
+   * @param {number} index
+   * @returns {boolean}
+   */
+  hasBlock(index) {
+    throw new Error(`hasBlock must be implemented`)
+  }
+
+  /**
+   * Gets the indexes of all blocks in the VHD.
+   * @returns {Array<number>}
+   */
+  getBlockIndexes() {
+    throw new Error(`getBlockIndexes must be implemented`)
+  }
+
+  /**
+   * determine the hash algorithm and hash length from https://en.wikipedia.org/wiki/Birthday_problem 
+   * compute the hash from the clear data 
+   * compute the path from the hash 
+   * split the path in 8 bytes parts at most , and create the directory tree if needed
+   * write the block in a temp place 
+   * rename the block to its defintive place, treat EEXIST as success ( block is already here )
+   * 
+   * @param {DiskBlock} diskBlock
+   * @return {Promise<number>} blockSize
+   */
+  async writeBlock(diskBlock) {
+    throw new Error(`writeBlock must be implemented`)
+  }
+
+  /**
+   * read the hash 
+   * compute the path from the hash 
+   * read the block from its path 
+   * @param {number} index
+   * @returns {Promise<DiskBlock>} diskBlock
+   */
+  async readBlock(index) {
+    throw new Error(`readBlock must be implemented`)
+  }
+
+  /**
+   * if block is already present in parent : not an issue, but still update the parent BAT
+   * @param {RemoteDisk} childDisk
+   * @param {number} index
+   * @param {boolean} isResumingMerge
+   * @returns {Promise<number>} blockSize
+   */
+  async mergeBlock(childDisk, index, isResumingMerge) {
+    throw new Error(`mergeBlock must be implemented`)
+  }
+
+  /**
+   * will need to also update he hashes 
+   * @param {Array<number>} blockIds
+   * @returns {Promise<void>}
+   */
+  async setAllocatedBlocks(blockIds) {
+    throw new Error(`setAllocatedBlocks must be implemented`)
+  }
+
+  /**
+   * write the metadata as a JSON flat file 
+   * @param {RemoteDisk} childDisk
+   * @returns {Promise<void>}
+   */
+  async flushMetadata(childDisk) {
+    throw new Error(`flushMetadata must be implemented`)
+  }
+
+
+  /**
+   * rename only the metadata file 
+   * delete overwritten target / disks if needed
+   * @param {string} newPath
+   */
+  async rename(newPath) {
+    throw new Error(`rename must be implemented`)
+  }
+
+  /**
+   * This must write the hashes and bat into a new files
+   * update the metadata that point to it
+   * then remove the older file
+   * 
+   * this is built on the fact that the merge now only
+   *  write it at the end of the job, so we don't update BAT
+   */
+  async flushMetadata(){
+
+  }
+
+
+  /**
+   * (for cleanVM)
+   * detecte older bat/hashes files 
+   */
+  async check(){
+
+  }
+
+}

@xen-orchestra/backups/disks/HashedDiskDeduplicated.mjs

@@ -0,0 +1,175 @@
+
+
+ class BlockStore{
+
+  constructor(handler, basePath){} 
+  /**
+   * write the block, returns the key in store
+   * @param {string} hash 
+   * @param {Buffer} content 
+   */
+    writeBlock( hash, content){
+    
+    }
+
+   /**
+   * return the code from its key 
+   * @param {string} hash 
+   * @param {Buffer} content 
+   */
+    readBlock( hash){
+
+    }
+ 
+   /**
+   * free one block use 
+   * @param {string} hash 
+   * @param {Buffer} content 
+   */
+    removeReference(hash){
+      
+    }
+ 
+   /**
+   * mark this bock as used
+   * @param {string} hash 
+   * @param {path} string 
+   */
+    addBlockReference(hash, path){
+      
+    }
+}
+
+ class BlockStoreFileSystem extends BlockStore{
+
+  constructor(handler, basePath){}
+  #handler
+    hash(diskBlock){}
+
+    #computePathInStore(hash){
+
+    }
+
+    /** 
+     * write the file in a temporary folder
+     * rename to the path computed from the hash/0
+     * treat EEXIST as success
+     * 
+     * @param {string} hash 
+     * @param {Buffer} content 
+     * @returns {Promise<size>} the size written on disk, 0 if if block already here
+     */
+    async writeBlock( hash, content){
+      try{
+        return await this.#handler.outputFile(this.#computePathInStore(hash), content)
+      }catch(err){
+        if(err.code === 'EEXIST'){
+          return 0
+        }
+        throw err
+      }
+    
+    }
+
+
+    /**
+     * compute the path from the hash 
+     * read the file from this path 
+     * @param {string} hash 
+     * @returns {Promise<Buffer} content 
+     */
+    readBlock( hash){
+      const path = this.#computePathInStore(hash)
+      return this.#handler.readFile(path)
+    }
+
+    /**
+     * remove the hard link
+     * check the hash/i and remove any that have a nLinks < 2 
+     * @param {*} hash 
+     */
+    removeReference(hash){
+      
+    }
+
+    /**
+     * nLinks is probably limited to 32 bits, but can be as low as 16 bits
+     * blocks are referenced once per disk 
+     * the chain should not be very long , even blocks present in all the disks like the empty block 
+     * 
+     * do a hard link from the hashed path
+     * to the path hash/0
+     * 
+     * ON EMLINK error ( too )
+     * try hash/1  ,.. hash/2 ... 
+     * if out of file copy the last then link from this new block
+     * @param {*} hash 
+     * @param {*} path 
+     */
+    addBlockReference(hash, path){
+      
+    }
+}
+
+
+ /**
+ * this class stores disk blocks in a global store at the root of the remote
+ * this class is more the second step after a complete HashedDisk implementation
+ * and should be  used in production instead of HashedDisk
+  */
+
+export class HashedDiskDeduplicated extends HashedDisk{
+  #blockStore
+  /**
+   * 
+   * @param {*} handler 
+   * @param {*} path 
+   * @param {BlockStore} blockStore 
+   */
+  constructor(handler, path, blockStore){
+
+  }
+
+  /**
+   * write the block into the block store, get the block key 
+   * link the block key to the same path as HashedDisk
+   * 
+   * if link return a ENOENT that means another process clean this block , re-add to block store
+   * 
+   * if this overwrite another block, then remove the old reference from the store 
+   * @param {DiskBlock} diskBlock 
+   */
+  async writeBlock(diskBlock) { 
+    let size
+    const path = this.computeBlockPath(diskBlock)
+    const hash = this.#blockStore.hash(diskBlock)
+    try{
+      // optimistically try if block exists
+      size = await this.#blockStore.addBlockReference(hash, path)
+      return size 
+    }catch(error){
+      // not really an issue , it was just that the block did not already exists
+    }
+    
+    size = await this.#blockStore.writeBlock(hash, diskBlock.data)
+    size += await this.#blockStore.addBlockReference(hash, path)
+    return size 
+  }
+
+  /** 
+   * remove all the references of this disk in the block store 
+   */
+   async unlink() {
+    throw new Error(`unlink must be implemented`)
+  }
+
+  /**
+   * or maybe another callback ? 
+   * call removeReference on each hash that is not present anymore in the parent after merge
+   * doing this once will limit the risks of race conditions when removing
+   */
+  flushMetadata(){
+
+  }
+
+}

@xen-orchestra/backups/disks/openDisk.mjs

@@ -0,0 +1,12 @@
+import { RemoteDisk } from "./RemoteDisk.mjs";
+
+
+/**
+ * open a disk on a remote , wether it's a RemoteVhd or a hashed disk
+ * refactor to use this method anywhere we use openVhd at least on @xen-orchestra/backups
+ * 
+ * @param {*} handler 
+ * @param {*} path
+ * @returns {Promise<RemoteDisk>} 
+ */
+export async function openDisk(handler,path){}