+#! /usr/bin/perl
+#
+# $Id$
+# Captive project doc Index page Perl template.
+# Copyright (C) 2003 Jan Kratochvil <project-www.jankratochvil.net@jankratochvil.net>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; exactly version 2 of June 1991 is required
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+
+package project::captive::doc::Index;
+require 5.6.0; # at least 'use warnings;' but we need some 5.6.0+ modules anyway
+use vars qw($VERSION $CVS_ID);
+$VERSION=do { my @r=(q$Revision$=~/\d+/g); sprintf "%d.".("%03d"x$#r),@r; };
+$CVS_ID=q$Id$;
+use strict;
+use warnings;
+
+BEGIN{ open F,"Makefile"; our $top_dir=pop @{[split /\s/,(grep /^top_srcdir/,<F>)[0]]}; eval "use lib '$top_dir'"; close F; }
+use My::Web;
+
+
+My::Web->init(
+ "__PACKAGE__"=>__PACKAGE__,
+ "title"=>'Captive NTFS doc',
+ "head_css"=>"
+.productname { font-family: cursive; }
+.fname { font-family: monospace; }
+.constant { font-family: monospace; }
+.author { font-family: cursive; }
+.stuff { font-style: italic; font-size: larger; margin-left: 20%; margin-right: 10%; }
+.function { font-family: monospace; }
+.type { font-family: monospace; }
+.command { font-family: monospace; }
+.instruction { font-style: italic; }
+",
+ );
+My::Web->heading();
+
+
+sub doc_img
+{
+my($img_base,$caption)=@_;
+
+ my $r="";
+ $r.='<table border="0" align="center">'."\n";
+ $r.="\t<tr><td>".img($img_base,$caption)."</td></tr>\n";
+ $r.="\t<caption>".CGI::escapeHTML($caption)."</caption>\n";
+ $r.='</table>'."\n";
+ return $r;
+}
+
+my $freespeech=a_href 'http://www.gnu.org/philosophy/free-sw.html','Free';
+my $freebeer=a_href 'http://www.gnu.org/philosophy/free-sw.html','free (as in beer)';
+
+sub productname
+{
+my($url,$name)=@_;
+
+ return '<span class="productname">'.a_href($url,CGI::escapeHTML($name)).'</span>';
+}
+my $Wine=productname 'http://www.winehq.com/','Wine';
+my $ReactOS=productname 'http://www.reactos.com/','ReactOS';
+my $LinuxNTFS=productname 'http://linux-ntfs.sourceforge.net/','Linux NTFS';
+my $GnomeVFS=productname 'http://developer.gnome.org/doc/API/gnome-vfs/','Gnome-VFS';
+my $GnomeVFSmodule=productname 'http://developer.gnome.org/doc/API/gnome-vfs/modules.html','Gnome-VFS-module';
+my $gnulinux='GNU/Linux';
+
+
+print vskip("10ex")."<h1 align=\"center\">!!! PRELIMINARY - TO BE UPDATED !!!</h1>\n".vskip("10ex");
+
+print <<"HERE";
+<h1>Abstract</h1>
+
+<p>Existing binary Microsoft Windows file system drivers were exploited
+for accessing drives with possibly proprietary file system data structures.
+Open file system API is provided to access these file system drivers.
+Microsoft Windows system components required by these drivers
+were analyzed and successfuly emulated in the GNU/Linux operating system.
+Currently the implementation allows applications running under the GNU/Linux
+operating system to access VFAT, ISO9660 and EXT2 drives. NTFS file system
+capability target is the final goal being currently developed on the base
+of this project assets.</p>
+
+
+<h1>Reasons for the Implementation</h1>
+
+ <p>Currently there is no possibility to any of the available $freespeech
+ ($freespeech used in the following text in the meaning of
+ "<a href="http://www.gnu.org/philosophy/free-sw.html">free as in speech</a>")
+ operating systems to reliably write to the most common disk partition
+ filesystem type - <span class="productname">Microsoft NTFS</span>. It would
+ be already supported a long time ago but there is no proper documentation of
+ <span class="productname">NTFS</span> filesystem data structures available.
+ Since <span class="productname">Microsoft</span> corporation continues in its
+ propagation of <span class="productname">Microsoft Windows NT</span>
+ (<span class="productname">NT</span> identifier used in the following text
+ applies to all the products of <span class="productname">Microsoft</span>
+ <span class="productname">NT</span> series such as
+ <span class="productname">NT 4.0</span>,
+ <span class="productname">2000</span> as NT-5.0
+ and
+ <span class="productname">XP</span> as NT-5.1.)
+ based operating systems <span class="productname">NTFS</span> is the default
+ disk file system type for new installations as described in the
+ <a href="http://www.microsoft.com/hwdev/tech/storage/ntfs-preinstallP.asp">recommendations
+ report</a> by <span class="productname">Microsoft</span>.</p>
+
+ <p>Unfortunately the <span class="productname">NTFS</span> filesystem has too
+ complex data structure to allow a complete reverse enginnering process in
+ reasonable time. Currently available $freespeech solutions such as $LinuxNTFS
+ filesystem have already implemented (more or less) reliable reverse
+ engineered read-only access. However <a name="reliability">the
+ reliability</a> of the read-write part of the access requires much better
+ knowledge of the <span class="productname">NTFS</span> data structures. Also
+ any future versions of <span class="productname">NTFS</span> filesystem would
+ require another major reverse engineering effort.</p>
+
+
+<h1>Goals of This Stage of the Project</h1>
+
+ <p>The <a name="NTFSgoal">ultimate goal</a> of this project is definitely the
+ free implementation of <a href="#reliability">reliable</a> read-write <span
+ class="productname">NTFS</span> filesystem driver. This project chose to
+ solve this problem in the style of $Wine project by using the original binary
+ <span class="fname">ntfs.sys</span> and emulating all the required layers of
+ <span class="productname">Microsoft Windows NT</span> for it.</p>
+
+ <p>Unfortunately this effort is tainted by only partial and generally
+ insufficient documentation of API between filesystem driver
+ (<span class="fname">ntfs.sys</span>) and the
+ <span class="productname">Microsoft Windows NT</span>
+ ("<a href="http://mail.gnu.org/archive/html/libtool/2000-09/msg00000.html">W32</a>"
+ in the following text) kernel <span class="fname">ntoskrnl.exe</span>. Note
+ that this API is a different than the one being used in the $Wine project
+ since <span class="productname">Wine</span> implements only the user space
+ part of W32.</p>
+
+ <p>There also exists a $freespeech
+ <span class="fname"><a href="http://sys.xiloo.com/projects/projects.htm#ext2fsd">ext2fsd.sys</a></span>
+ W32 filesystem driver for <span class="constant">ext2</span> filesystems with
+ source files freely available for it. Moreover original
+ <span class="productname">Microsoft Windows NT</span> filesystems
+ <span class="fname">cdfs.sys</span> and
+ <span class="fname">fastfat.sys</span> (which correspond to Linux
+ <span class="productname">iso9660</span> and
+ <span class="productname">vfat</span> filesystems, resp.) are easy enough to
+ get working in reasonable time. All these filesystem drivers also use only
+ the documented filesystem data structures which makes their behaviour better
+ controllable when debugging the project.</p>
+
+ <p>Therefore this stage of the project is intended to get only the original
+ W32 binary form of <span class="fname">cdfs.sys</span> and
+ <span class="fname">fastfat.sys</span> drivers working. This goal was
+ achieved and the compatibility with <span class="fname">ext2fsd.sys</span>
+ can be considered as an additional benefit.</p>
+
+
+<h1>Architecture</h1>
+
+ <p>Although this project attempts to be as general and crossplatform as
+ possible to avoid being needlessly bound by any resources the current
+ implementation is being developed/tested on $gnulinux. The principle of the
+ project lies in the glue between
+ <span class="productname">Microsoft Windows NT</span> kernel space
+ environment and $gnulinux user space process environment. Currently there are
+ no plans to ever extend the project's crossplatformity beyond the
+ <span class="constant">i386</span> processor
+ (<span class="constant">i386</span> used here as
+ <a href="http://www.intel.com/">Intel</a> architecture covering 32-bit
+ processors compatible with <span class="constant">i386</span>,
+ <span class="constant">i486</span>, ...).
+
+ <a name="existing_emulation"><h2>Existing Emulation Projects</h2></a>
+
+ <p>There are two well-known $freespeech projects emulating W32 subsystems
+ to reach the compatibility with various W32 components:
+ $Wine and $ReactOS. Sad moment is that the goals of this project do not fit
+ very well into any role in those two projects:
+
+ <table align="center" border="1">
+ <tr>
+ <th><a href="#guestosnote">Guest-OS</a></th>
+ <th><a href="#hostosnote" >Host-OS</a ></th>
+ <th>Implements</th>
+ <th>W32 kernel library</th>
+ </tr>
+ <tr>
+ <td>$Wine</td>
+ <td>$gnulinux</td>
+ <td>W32 user space</td>
+ <td><span class="fname">ntdll.dll</span></td>
+ </tr>
+ <tr>
+ <td>$ReactOS</td>
+ <td><span class="constant">i386</span> hardware</td>
+ <td>W32 kernel and user space</td>
+ <td><span class="fname">ntoskrnl.exe</span></td>
+ </tr>
+ <caption>Existing Emulation Projects Characteristics</caption>
+ </table>
+
+ <dl>
+ <a name="guestosnote"><dt>Guest-OS</dt></a>
+ <dd><a href="http://www.vmware.com/support/reference/common/glossary/#guestos">Guest OS</a>:
+ An operating system that runs inside a virtual machine.</dd>
+ <a name="hostosnote" ><dt>Host OS</dt></a>
+ <dd><a href="http://www.vmware.com/support/reference/common/glossary/#hostos" >Host OS</a>:
+ An operating system that runs on the host machine.</dd>
+ </dl>
+
+ <p>While $ReactOS provides the necessary W32 kernel subsystem emulation
+ code we also need to run such <a href="#guestosnote">Guest-OS</a> in the <a
+ href="#hostosnote">Host-OS</a> $gnulinux. Initially it was planned to
+ extend $Wine with the W32 kernel space emulation functionality but
+ fortunately <span class="author">Steven Edwards</span> pointed to the $ReactOS
+ which better suits the needs of this project by its already implemented W32
+ kernel space emulation.</p>
+
+ <p>The <a name="reactos_nocare">original reasons</a> for developing
+ $ReactOS still make no sense to the author of this project. Free
+ implementation of W32 platform standalone running on the machine hardware
+ is no longer free as most od the W32 applications are usually closed source
+ and the user still looses its freedom on the application level anyway. Even
+ in the case of available free applications there still remains the
+ disadvantage of loosing the Host-OS platform availability if implemented in
+ the $Wine style. For these ideology incompatibilities not much effort was
+ made for acceptance the fixes and improvements of $ReactOS by this project.
+ Moreover new functionality is not being implemented to the $ReactOS part
+ but it is coded in Gnome style in the project specific source files
+ place.</p>
+
+ <p>The most serious problem of $ReactOS is its dependence on the direct
+ <span class="constant">i386</span> hardware instead of some
+ <a href="#hostosnote">Host-OS</a> as required by the goals of this project.
+ W32 is designed to be hardware-independent using its
+ <span class="fname">hal.dll</span>. Unfortunately $ReactOS does not follow
+ this design and thus there are needed various patches and replaces of its
+ various parts and its hardware-dependent code. Despite it $ReactOS code
+ base still made a big asset for this project.</p>
+
+
+
+
+
+ <p>Some API functions are provided both by
+ <span class="fname">ntdll.dll</span> and
+ <span class="fname">ntoskrnl.exe</span> in W32.
+ <span class="author">Casper Hornstrup</span> enlightened such functions
+ calling conventions have to be differentiated as
+ <span class="fname">ntdll.dll</span> lives in the user space (low address
+ space -- below <span class="constant">0x80000000</span>) and
+ <span class="fname">ntoskrnl.exe</span> in the kernel space (high address
+ space -- above <span class="constant">0x80000000</span>). Although they
+ contain slightly different set of symbols (functions)
+ <span class="fname">ntdll.dll</span> still can be considered as a user
+ space interface to the kernel space implementation by
+ <span class="fname">ntoskrnl.exe</span>.</p>
+
+ <h2>API Function Implementation Choices</h2>
+
+ <p>During the initial point of the project development all the API
+ functions were defined as unimplemented, of course. Any call of such
+ unimplemented function is fatal and results in program termination. When we
+ need to implement any required API function we have multiple choices to do
+ so:
+ <a href="#functype_pass">Direct pass to original
+ <span class="fname">ntoskrnl.exe</span></a>,
+ <a href="#functype_wrap">Wrap of the original
+ <span class="fname">ntoskrnl.exe</span> function</a>,
+ <a href="#functype_native_reactos">Native implementation -- $ReactOS,
+ <a href="#functype_native_wine">Native implementation -- $Wine
+ or
+ <a href="#functype_native_libcaptive">Native implementation
+ -- project specific</a>.
+ <!-- a href="#functype_undef" Undefined function /a -->
+
+ <h2>"patched" vs. "unpatched" Libraries</h2>
+
+ <p>Library is called <span class="constant">patched</span> if we require
+ loading its original binary code file. Project needs to patch it to be able
+ to trap all the function entry points. The typical current
+ <span class="constant">patched</span> library of this project is
+ <span class="fname">ntoskrnl.exe</span>.</p>
+
+ <p>Library is called <span class="constant">unpatched</span> if no original
+ binary code is needed since all of its functions are completely emulated by
+ <a href="#functype_native">the native implementations</a> of this project.
+ The typical <span class="constant">unpatched</span> representative is
+ <span class="fname">hal.dll</span> as it specializes on the hardware
+ dependent code and therefore it must be completely replaced by this project
+ running in the $gnulinux operating system environment. Early versions of
+ this project had also full <span class="constant">unpatched</span>
+ <a href="#native_ntoskrnl">native implementation of
+ <span class="fname">ntoskrnl.exe</span></a> but it no longer applies.</p>
+
+ <h2>Memory Management</h2>
+
+ <p>Original <span class="productname">Microsoft Windows NT</span>
+ architecture uses two address space areas - user space and kernel space.
+ User space is mapped in the range <span class="constant">0x00000000</span>
+ to <span class="constant">0x7FFFFFFF</span>, kernel space is mapped in the
+ range <span class="constant">0x80000000</span>
+ (<span class="constant">KERNEL_BASE</span> in $ReactOS sources) to
+ <span class="constant">0xFFFFFFFF</span>. All these virtual memory ranges
+ represent addresses after their MMU (Memory Management Unit) mapping, of
+ course. More discussion can be found in the
+ <a href="http://www.microsoft.com/hwdev/platform/server/PAE/PAEmem.asp">description
+ by <span class="productname">Microsoft</span></a>.</p>
+
+ <p>This project runs in the virtual address space used both for the UNIX
+ user space process part and for the W32 kernel space. Therefore this
+ project defines that W32 kernel runs in the whole range
+ <span class="constant">0x00000000</span> to
+ <span class="constant">0xFFFFFFFF</span> since there are no special mapping
+ assumptions about the UNIX user space process mapping. No W32 user space
+ exists in this project. Such approach also nullifies any special memory
+ moving operations between W32 kernel space and W32 user space memory areas
+ (such as <span class="function">MmSafeCopyToUser()</span>).</p>
+
+ <h2>Unicode Strings and Characters</h2>
+
+ <p>W32 platform uses 16-bit type <span class="type">wchar_t</span> while $gnulinux uses a
+ 32-bit one. This can be problem during GCC (GNU C Compiler)
+ compilation of combination of native UNIX C sources (assuming 32-bit
+ GCC with 32-bit <span class="type">wchar_t</span>) and
+ $ReactOS C sources (assuming W32 compiler with 16-bit
+ <span class="type">wchar_t</span>) for literal wide strings
+ (C source file systax: <span class="command">L"wstring"</span>).
+ Possibilities to solve this issue list:</p>
+
+ <ul>
+ <li>
+ <p>Using <span class="constant">-fshort-wchar</span> GCC option and
+ strictly differentiate between compilation of
+ <span class="productname">ReactOS</span> code and UNIX code.</p>
+
+ <p>pros: No source modifications needed, no runtime performance hit.</p>
+
+ <p>cons: No type checking if some part of code has bad compilation
+ flags, complicated way to completely split
+ <span class="productname">ReactOS</span> and UNIX code.</p>
+ </li>
+ <li>
+ <p>Wrap all <span class="productname">ReactOS</span> literal constants
+ by some conversions function call (implemented as macro
+ <span class="function">REACTOS_UCS2()</span> by this project).</p>
+
+ <p>pros: Any forgotten/mistaken conversions are type-checked and warned
+ during the compilation by GCC.</p>
+
+ <p>cons: All compiled <span class="productname">ReactOS</span> sources
+ files containing literal wide strings have to be wrapped/modified,
+ performance hit by runtime string conversions.</p>
+
+ <p>This solution was chosen to get the internal sanity checking
+ benefit.</p>
+ </li>
+ </ul>
+
+ <h2>Supported Binary Formats</h2>
+
+ <p>The native W32 binary format is identified as
+ <span class="constant">PE-32</span> (Portable Executable 32-bit), such
+ files have all the usual extensions such as
+ <span class="fname">.sys</span>, <span class="fname">.exe</span>,
+ <span class="fname">.dll</span> etc. <span class="constant">PE-32</span>
+ loading support was already implemented by $ReactOS, its memory mapping
+ specifics just had to be ported to $gnulinux environment by this project.
+ This loading support does not (yet) cover importing of debug symbols from
+ W32 <span class="fname">.PDB</span> (Program DataBase) files in $gnulinux
+ ABI (Application Binary Interface) compatible way.</p>
+
+ <p>This project also supports transparent loading of UNIX
+ <span class="fname">.so</span> (Shared Object file) binary format. If you
+ have W32 source files for some W32 library you can try to compile it by GCC
+ to get the shared library with $gnulinux ABI compatible debug information
+ (GCC option <span class="constant">-ggdb3</span> recommended). Beware of
+ possible compilation problems as <span class="productname">Microsoft</span>
+ C code expects <span class="constant">exception</span> handling to be
+ supported by the compiler (definitely not the case of the plain C compiler
+ of GCC) --- all the exception catching code should be discarded as any
+ <a href="#exception_fatal">generated exceptions are always fatal</a> when
+ such driver is running in the scope of this project.</p>
+
+ <p>Be aware of some differences if you use
+ <span class="constant">PE-32</span> binary format file vs.
+ <span class="fname">.so</span> format file.
+ <span class="constant">PE-32</span> use the appropriate W32 specific
+ <a href="#calltype">cdecl/stdcall/fastcall call types</a>,
+ <span class="fname">.so</span> must be completely compiled in the standard
+ UNIX <a href="#calltype_cdecl">cdecl call type semantics</a>.
+ <a href="#functype_native">Native function implementations</a> do not need
+ to be explicitely exported by <span class="fname">captivesym</span> as they
+ are resolved automatically by the UNIX dynamic system linker. It may be
+ surprising you will have to fix all such missing symbol exports if you
+ advance during the development from the debugging
+ <span class="fname">.so</span> file for the production version of the
+ original <span class="constant">PE-32</span> binary file.</p>
+
+ <h2>Reverse Engineering</h2>
+
+ <p>This project has no intentions to reverse engineer and document the
+ filesystem data structures themselves since they are being encapsulated by
+ the filesystem driver. For these reasons the resources available in
+ projects such as $LinuxNTFS get out of any possible use. This project goal
+ is to provide fully compatible API interface to the rest of the W32 system
+ to persuade the filesystem driver it is running in the native
+ <span class="productname">Microsoft Windows XP</span> environment.</p>
+
+ <p>All the W32 filesystem drivers are running in the W32 kernel address
+ space and this area of W32 API is not much documented by
+ <span class="productname">Microsoft</span>. Some API functions are not
+ documented at all and the others are documented insufficiently for a their
+ possibly needed reimplementation from scratch. Documentation being
+ consulted primarily consists of
+ <span class="productname"><a href="http://msdn.microsoft.com/library/default.asp?url=/library/en-us/kmarch/hh/kmarch/kmhdr_6enb.asp">MSDN (Microsoft Developer Network) Kernel-Mode Driver Architecture: Windows DDK</a></span>
+ documentation and also various other 3rd party documentation resources such as
+ <span class="productname"><a href="http://www.osr.com/ntinsider/1996/cacheman.htm">The NT Cache Manager Description</a></span>,
+ <span class="productname"><a href="http://www.winntmag.com/Articles/Print.cfm?ArticleID=3864">Learn About NT's File-system Cache</a></span>,
+ <span class="productname"><a href="http://www.ntfsd.org/archive/">NT File System Developers mailing list archives</a></span>
+ including various
+ <a href="http://www.google.com/search?q=site%3Amicrosoft.com">fulltext searches</a>
+ through Internet from case to case.</p>
+
+ <p>Sometimes no sufficient documentation was found and some code behaviour
+ had to be reverse engineered directly from the binaries of
+ <span class="fname">ntoskrnl.exe</span>,
+ <span class="fname">cdfs.sys</span>
+ and/or
+ <span class="fname">fastfat.sys</span>.
+ Up to now the code was disassembled by
+ <span class="productname"><a href="http://www.simtel.net/pub/pd/29498.html">IDA Freeware</a></span>
+ and by
+ <span class="productname">dumpbin.exe</span> of
+ <span class="productname">Microsoft Visual Studio</span>.
+ <span class="productname">dumpbin.exe</span> is fortunately able to
+ interpret debug symbols from W32 <span class="fname">.PDB</span>
+ (Program DataBase) debug information files.</p>
+
+ <a name="law"><h2>Laws and Licensing Conditions</h2></a>
+
+ <p>If you are an <span class="productname">authorized user</span> of
+ <span class="productname">Microsoft Windows NT</span> the laws in some
+ countries give you the right to fully handle the product in any way you
+ want. Therefore you can disassemble the product even in the case you had
+ to agree with the product license forbidding such disassembly as the
+ country laws override any such license agreement.</p>
+
+ <h3>Microsoft Service Pack</h3>
+
+ <p>Sometimes you may have the legal license for
+ <span class="productname">Microsoft Windows NT</span>
+ but for various technical reasons you do not have the media and/or
+ installation ready at the place of intended use of this project.</p>
+
+ <p>Fortunately <span class="productname">Microsoft</span> provides
+ $freebeer update packages for its
+ <span class="productname">Microsoft Windows</span> products called
+ <span class="productname">Service Packs</span>; the latest one is
+ <span class="productname"><a href="http://www.microsoft.com/WindowsXP/pro/downloads/servicepacks/sp1/checkedbuild.asp">Microsoft Windows XP Service Pack 1a</a></span>.</p>
+
+ <p>This downloadable file contains the full versions of the essential
+ files needed for the current stage of this product:
+ <span class="fname">cdfs.sys</span>,
+ <span class="fname">fastfat.sys</span>
+ and
+ <span class="fname">ntoskrnl.exe</span>.
+ It even contains <span class="fname">ntfs.sys</span> for the planned
+ <a href="#NTFSgoal"><span class="productname">NTFS</span>
+ functionality</a>.</p>
+
+ <p><span class="productname">Service Pack</span> also contains
+ EULA (End User License Agreement) paper disallowing any use of
+ <span class="productname">Service Pack</span> outside its original
+ intentions. According to the laws of some countries you need to be
+ <span class="productname">authorized user</span> of the
+ <span class="productname">Microsoft Windows XP</span> product to be
+ allowed to use the files contained in such
+ <span class="productname">Service Pack</span> without the bindings of its
+ EULA. Even the interpretation of such laws may vary.</p>
+
+ <p>It would be a breach of the law by the project author to provide
+ automatic (=hidden) functionality to download and extract the
+ <span class="productname">Service Pack</span> files. On the other hand it
+ is perfectly legal to ask user for his/her confirmation whether he/she is
+ really the <span class="productname">authorized user</span> of
+ <span class="productname">Microsoft Windows XP</span> product and
+ download/extract the <span class="productname">Service Pack</span> files
+ accordingly.</p>
+
+ <h2>Project Architecture</h2>
+
+ @{[ doc_img 'fig/architecture','Project Architecture' ]}
+
+ <p>Most of the work of this project is located in the single box called
+ "<span class="constant">libcaptive</span>" located in the center
+ of the scheme. This component implements the core W32 kernel API by
+ <a href="#functype">various methods described in this document</a>.
+ The "<span class="constant">libcaptive</span>" box cannot be
+ further dissected as it is just an implementation of a set of API
+ functions. It could be separated to several subsystems such as the Cache
+ Manager, Memory Manager, Object Manager, Runtime Library, I/O Manager
+ etc. but they have no interesting referencing structure.</p>
+
+ <p>As this project is in fact just a filesystem implementation every
+ story must begin at the device file and end at the filesystem operations
+ interface. The unified suppported interfaces are
+ <span class="productname"><a href="http://developer.gnome.org/doc/API/2.0/glib/">GLib</a></span>
+ (the most low level portability, data-types and utility library for Gnome)
+ <span class="type">GIOChannel</span> (for the device access) and the custom
+ <span class="constant">libcaptive</span> filesystme API. Each of these ends
+ can be connected either to some direct interface (such as the
+ <span class="constant">captive-cmdline</span> client) or it can connected
+ as a general $GnomeVFS filter. $GnomeVFS offers nice filter interface on
+ the UNIX user-privileges level for transparent operation with archives and
+ network protocols. This filter interface was used by this project to turn
+ the device reference such as <span class="fname">/dev/hda3</span> or
+ <span class="fname">/dev/discs/disc0/part3</span> to the fully accessible
+ filesystem (pretending being an "archive" in the device
+ reference). This device access can be specified by $GnomeVFS URLs such as:
+ <span class="fname">file:///dev/hda3#captive-fastfat:/autoexec.bat</span></p>
+
+ <p>If the passed device reference is requested by the user to be accessed
+ either in <span class="dashdash">--ro</span> (read-only) mode or in the
+ <span class="dashdash">--rw</span> (full read-write) mode there are no
+ further device layers needed. Just in the case of
+ <span class="dashdash">--blind</span> mode another layer is involved to
+ emulate read-write device on top of the real read-only device by the method
+ of non-persistent memory buffering of all the possible write requests.</p>
+
+ <p>Such device is still only a UNIX style GLib <span
+ class="type">GIOChannel</span> type at this point. As we need to supply it
+ to the W32 filesystem driver we must convert it to the W32 I/O Device
+ with its capability of handling <span class="type">IRP</span>
+ (<span class="constant">I/O Request Packet</span>; structure holding the
+ request and result data for any W32 filesystem or W32 block device
+ operation)
+ requests from its upper W32 filesystem driver. Such W32 I/O Device can
+ represent either <span class="type">CD-ROM</span> or
+ <span class="type">disk</span> device type as different W32 filesystem
+ drivers require different media types:</p>
+
+ <h3>cdfs.sys</h3>
+
+ <p><span class="type">CD-ROM</span> filesystem runs just on the
+ <span class="constant">FILE_DEVICE_CD_ROM_FILE_SYSTEM</span> device type.
+ Use <span class="dashdash">--cdrom</span> option of this project for
+ <span class="fname">cdfs.sys</span>.</p>
+
+ <h3>fastfat.sys</h3>
+
+ <p><span class="type">FAT</span> filesystem supports both the (expected)
+ <span class="constant">FILE_DEVICE_DISK_FILE_SYSTEM</span> device type
+ but it also supports the reading of
+ <span class="constant">FILE_DEVICE_CD_ROM_FILE_SYSTEM</span> devices as
+ you can use <span class="type">FAT</span> filesystem on <span
+ class="type">CD-ROM</span> media in W32 environment. It is recommended to
+ use <span class="dashdash">--disk</span> option of this project for
+ <span class="fname">fastfat.sys</span>.</p>
+
+ <h3>ext2fsd.sys</h3>
+
+ <p><span class="type">ext2</span> filesystem supports just the
+ <span class="constant">FILE_DEVICE_DISK_FILE_SYSTEM</span> device type.
+ Use <span class="dashdash">--disk</span> option of this project for
+ <span class="fname">ext2fsd.sys</span>.</p>
+
+ @{[ vskip("3ex") ]}
+
+ <p>W32 media I/O Device is accessed from the W32 filesystem driver.
+ The filesystem driver itself always creates volume object by
+ <span class="function">IoCreateStreamFileObject()</span> representing the
+ underlying W32 media I/O Device as the object handled by the
+ filesystem driver itself. All the client application filesystem requests
+ must be first resolved at the filesystem structures level, passed to the
+ volume stream object of the same filesystem and then finally passed to the
+ W32 media I/O Device (already implemented by this project as an
+ interface to <span class="type">GIOChannel</span> noted above).</p>
+
+ <p>The filesystem driver is called by the core W32 kernel implementation of
+ <span class="constant">libcaptive</span> in
+ <a href="#synchronous">synchronous way</a> in single-shot manner instead of
+ the several reentrancies while waiting for the disk I/O completions as can
+ be seen in the original
+ <span class="productname">Microsoft Windows NT</span>.
+ This single-shot synchronous behaviour is possible since all the needed
+ resources (disk blocks etc.) can be always presented as instantly ready as
+ their acquirement is solved by <a href="hostosnote">Host-OS</a> outside of
+ the W32 emulated <a href="guestosnote">Guest-OS</a> environment.</p>
+
+ <p><span class="constant">libcaptive</span> offers the W32 kernel
+ filesystem API to the upper layers. This is still not the API the common
+ W32 applications are used to as they use W32 libraries which in turn pass
+ the call to W32 kernel. For example
+ <span class="function">CreateFileA()</span> is being implemented by several
+ libraries such as <span class="fname">user32.dll</span> as a relay
+ interface for the kernel function
+ <span class="function">IoCreateFile()</span> implemented by this
+ project's <span class="constant">libcaptive</span> W32 kernel
+ emulation component.</p>
+
+ <p>As it would be very inconvenient to use the legacy, bloated and UNIX
+ style unfriendly W32 kernel filesystem API this project offers its own
+ <a href="#client_interface">custom filesystem API interface</a> inspired by
+ the $GnomeVFS client interface adapted to the specifics of W32 kernel API.
+ This interface is supposed to be easily utilized by
+ <a href="#client_interface_customapp">a custom application accessing
+ the W32 filesystem driver</a>.</p>
+
+ <p>The rest of the story is not much special for this project since this is
+ a common UNIX problem how to offer user space implemented UNIX filesystem
+ as a generic system filesystem (as those are usually implemented only as
+ the components od UNIX kernel). The most thin implementation would be to
+ implement <FIXME:LUFS><a href="#fuse_interface">FUSE \bookcitation{FUSE}
+ (Filesystem in Userspace project for $gnulinux implemented by its own
+ filesystem code for Linux kernel)
+ interface</a> for the purpose but such feature is not yet implemented.
+ Currently this project implements
+ <a href="#offered_gnomevfs">Gnome-VFS interface</a> allowing its filesystem
+ access even without any involvement of UNIX kernel from any
+ $GnomeVFS aware client application (such as
+ <span class="fname">gnome-vfs/tests/test-shell</span>).
+ This <a href="#offered_gnomevfs">Gnome-VFS interface</a> connects the data
+ flow of this project in two points - both as the lowest layer device image
+ source and also as the upper layer for the filesystem operation
+ requests.</p>
+
+ <p>That's all folks!</p>
+
+ <a name="mounted_one"><h2>At Most One Mounted Filesystem</h2></a>
+
+ @{[ doc_img 'fig/sandbox','Multiple Filesystems by libcaptive Sandboxing' ]}
+
+ <p>The project technically supports only one (exactly one...) mounted
+ filesystem device and only one filesystem driver. There is nothing
+ complicated to support multiple disks and multiple loaded filesystem
+ modules but as they would share the address space it would only bring
+ a possible complications during bug reports and the bug solving
+ itself. It was considered as a more sane way to support multiple W32
+ mounted disks by completely separately running project instances in
+ a different UNIX processes communicating from their sandboxes via
+ <a href="#todo_sandbox">CORBA sandbox interface</a>. This sandboxing
+ feature is not yet deployed although its code is already prepared.</p>
+
+ <p>The project also does not support any state cleanup to be able to load
+ filesystem <span class="constant">A</span>,
+ cleanup <span class="constant">A</span> and load a different
+ filesystem <span class="constant">B</span> in the same process address
+ space. It complies with the preventions of the possible debugging
+ complications as noted above. Despite this you still must call the function
+ <span class="function">captive_shutdown()</span> to flush all the pending
+ filesystem buffers to the disk. After calling
+ <span class="function">captive_shutdown()</span> the process address space is
+ no longer usable for any further project operations and the process is
+ expected to be terminated in the manner compatible with its driving
+ <a href="#todo_sandbox">CORBA sandbox interface</a> control master.</p>
+
+ <p>Each sandbox executing the untrusted W32 binary filesystem driver code
+ is connected through its
+ <a href="#todo_sandbox">CORBA sandbox interface</a> at the point of upper
+ layer <span class="constant">libcaptive</span>-specific filesystem API, at
+ the point of the bottom layer of <span class="type">GIOChannel</span>
+ device access and also for transfers of GLib logging
+ messages/warnings/errors out of the sandbox to the user.</p>
+
+
+<h1>Choice of the Emulation Methods</h1>
+
+ <p>The intent of the project was to get reliable read-write access to
+ <span class="productname">NTFS</span> partition. There are several possible
+ ways to achieve that:</p>
+
+ <h2>Virtualmachine Running the Original W32 Subsystem</h2>
+
+ <p>Creating virtual-hardware PC and running the original W32 binaries
+ including their boot-loader etc. Disk device access would be passed as
+ virtual IDE disk (=hard disk drive). File access API would be implemented
+ either by special escaping by some trapped instruction out of the
+ virtualmachine while using W32 file access API or using the standard W32
+ SMB (Server Message Block) network access through some virtual network
+ card. The latter network access solution is almost the currently available
+ possibility of running full-blown disk-sharing real
+ <span class="productname">Microsoft Windows NT</span> inside virtual
+ machine emulator such as <span class="productname">VMware</span>.</p>
+
+ <p>pros: Full compatibility due to fully native codebase.</p>
+
+ <p>cons: Hard to debug, missing documentation of NT booting internals,
+ possible problems by different PC virtual-hardware than expected by NT,
+ requirement of fully installed
+ <span class="productname">Microsoft Windows NT</span> product.</p>
+
+ <a name="method_ntoskrnl"><h2>"ntoskrnl.exe" Inside Virtual Address Space</h2></a>
+
+ <p>This solution was chosen by the project. Binary filesystem driver and
+ also <span class="fname">ntoskrnl.exe</span> binary file are required.
+ Unfortunately <span class="fname">ntoskrnl.exe</span> expects a native
+ PC virtual-hardware missing during regular UNIX user space process
+ emulation, therefore such instructions must be trapped and emulated/ignored
+ from case to case.</p>
+
+ <p>Also the <a name="init_ntoskrnl">initialization code of <span
+ class="fname">ntoskrnl.exe</span></a> is not executed by this project since
+ it expects to get full PC hardware access privileges and thus some
+ datastructures do not get initialized by it (need to be trapped later at
+ runtime stage). Some of the missing initializations are solved by
+ <a href="#functype_wrap">API functions wrapping</a>.
+
+ <p>pros: Lightweight, easier to debug.</p>
+
+ <p>cons: Possible incompatible emulation of
+ <span class="fname">ntoskrnl.exe</span> parts, missing documentation needed
+ for the implementation.</p>
+
+ <h2>Filesystem Driver Inside Virtual Address Space</h2>
+
+ <p>Unlike <a href="#method_ntoskrnl">previous method</a> here we do not use
+ even <span class="fname">ntoskrnl.exe</span> as the complete kernel part of
+ W32 is <a name="native_ntoskrnl">emulated from the project source
+ files</a>. <span class="fname">cdfs.sys</span> driver was successfuly ran
+ in this manner in the former versions of this project but the possibility
+ to run without <span class="fname">ntoskrnl.exe</span> was dropped since it
+ had no licensing gains (you need the original
+ <span class="productname">Microsoft Windows NT</span> files at least for
+ the filesystem driver itself) and the emulation of undocumented parts
+ reusable from <span class="fname">ntoskrnl.exe</span> binary was
+ a pain.</p>
+
+ <p>pros: Lightweight, easier to debug.</p>
+
+ <p>cons: Possible incompatible emulation of the whole
+ <span class="fname">ntoskrnl.exe</span>, its missing documentation.</p>
+
+
+<h1>Implementation Details</h1>
+
+ <a name="functype"><h2>API Function Implementation Choices</h2></a>
+
+ <p>For each function exported by W32
+ <span class="fname">ntoskrnl.exe</span> and imported and called by the
+ filesystem driver a decision needs to be made to properly implement its
+ functionality. Currently implemented functionality statistics are provided
+ below:</p>
+
+ <FIXME:numbers>
+ <table border="1" align="center">
+ <tr><th>Function type </th><th>Items</th><th>Portion</th></tr>
+ <tr><td><a href="#functype_pass">pass</a> </td><td> 46</td><td> 21%</td></tr>
+ <tr><td><a href="#functype_wrap">wrap</a> </td><td> 1</td><td> 0%</td></tr>
+ <tr><td><a href="#functype_native_reactos">native-ReactOS</a></td><td> 94</td><td> 43%</td></tr>
+ <tr><td><a href="#functype_native_libcaptive">native-own</a> </td><td> 79</td><td> 36%</td></tr>
+ <caption>Function Implementation Types Statistics</caption>
+ </table>
+
+ <p>As there are several choices to implement each function the usual
+ attempts/investigations ordering is listed in the sections below.</p>
+
+ <p>Special case must be taken for data-type symbols since they are
+ referenced without the possibility of catching the code flow by some
+ breakpoints (it would be possible only in some special access cases). Data
+ export symbols of <span class="constant">unpatched</span> libraries must
+ contain already prepared content at the runtime. There is a problem
+ with <span class="constant">patched</span> libraries where it is necessary
+ to also fully implement the data symbol as
+ <a href="#functype_native">native implementation</a> since there is no
+ possibility to <a href="#functype_pass">pass</a> the data symbol instead of
+ the original W32 data location and therefore there will be two instances of
+ such data variable place. As there will be also the uncaught references for
+ such W32 data location from the <span class="constant">patched</span>
+ library itself such symbols should be usually only some constants (such as
+ <span class="constant">KeNumberProcessors</span>).</p>
+
+ <p>W32 platform symbols export/import can be based either on the symbol
+ name itself or it can be also exported and imported just by its
+ identification number called <span class="constant">Ordinal</span>.
+ Although it saves some jumptables file binary size it is currently no
+ longer used by W32 binaries and this project also does not support such
+ <span class="constant">Ordinal</span> symbol reference type at all.</p>
+
+ <p>All the exporting magic is handled by custom script
+ <span class="fname">captivesym</span> processing the definition file
+ <FIXME:span class="fname">src/libcaptive/ke/exports.captivesym</span> to produce
+ the intermediate relaying code
+ <FIXME:span class="fname">src/libcaptive/ke/exports.c</span>. For details of the
+ <span class="fname">captivesym</span>-specific source file syntax please
+ see its documentation: <FIXME:span class="fname">doc/captivesym-pod.html</span>
+
+ <a name="functype_pass"><h3>Direct Pass to Original "ntoskrnl.exe"</h3></a>
+
+ <p>Simple (standalone) functions such as
+ <span class="function">RtlTimeToSecondsSince1970()</span> can be simply
+ passed to the original implementation in
+ <span class="fname">ntoskrnl.exe</span> as they make no hardware access
+ and they do not expect any special internal data structures to be set up
+ in advance by an earlier library initialization. A common case are all
+ the data structures utility functions such as
+ <span class="constant">GenericTable</span> subsystem or
+ <span class="constant">LargeMcb</span> handling.</p>
+
+ <a name="functype_pass_fromunix"><h4>Pass from UNIX Code</h4></a>
+
+ <p>Control flow begins in some standard UNIX code. Such code is always
+ using <a href="#calltype_cdecl">cdecl call type</a> for all its
+ intracalls. <a href="#functype_native_reactos">Native functions
+ compiled from <span class="productname">ReactOS</span> sources</a> use
+ their own <a href="#calltype">cdecl/stdcall/fastcall</a> declarations
+ but these call type modifications are discarded during compilation for
+ this project by the <span class="constant">LIBCAPTIVE</span>
+ symbol.</p>
+
+ <p>UNIX code calls <span class="function">FUNCTIONNAME()</span> relay
+ from the generated UNIX jump table. Such relay will debug dump the
+ passed arguments and finally pass the control to the original W32
+ function code in the proper call type
+ <a href="#calltype">cdecl/stdcall/fastcall</a> for a given
+ function.</p>
+
+ <p>Original W32 code entry point is always trapped by a breakpoint
+ although it would not be needed during this specific direct pass from
+ UNIX code to the original W32 implementation. Still the breakpoint has
+ to be there to catch some other (such as intra-W32) possible calls
+ described later. There are several more ways to define breakpoint in
+ the code. One way is to use processor hardware breakpoint support but
+ the number of breakpoints is limited. The other way is to patch in the
+ <span class="instruction">@{[ 'int $3' ]}</span> instruction but it will invoke
+ <span class="constant">SIGTRAP</span> signal handler conflicting with
+ the possible debugger (<span class="productname">gdb(1)</span>)
+ control. This project uses the <span class="instruction">hlt</span>
+ instruction, which also has a single-byte opcode as
+ <span class="instruction">@{[ 'int $3' ]}</span> and it is a privileged
+ instruction forbidden to be used from the UNIX user space code.
+ <span class="instruction">hlt</span> invokes
+ <span class="constant">SIGSEGV</span> signal which can be resolved by
+ a custom signal handler without any conflict with the possible
+ debugger control; <span class="productname">gdb(1)</span> needs the
+ following command to pass through such
+ <span class="constant">SIGSEGV</span> signal:</p>
+
+ <blockquote class="command">
+ <p>handle SIGSEGV nostop noprint pass</p>
+ </blockquote>
+
+ <p>When a breakpoint gets caught, we usually need to return to the
+ running code. Unfortunately it is not possible because of the patched
+ breakpoint opcode. The breakpoint cannot be simply removed upon return
+ as it would permanently loose control over the point of entry. Even if
+ the return would include faking of the return address in the bottom
+ stack frame to patch the breakpoint back during later function exit it
+ still would not solve the caughts of inner calls of recursive
+ functions. One of the working possibilities would be to patch the
+ original instruction back and perform a singlestep provided by
+ <span class="function">ptrace(2)</span> syscall. However such
+ singlestep needs another controlling UNIX process and it would again
+ conflict with the debuggers such as
+ <span class="productname">gdb(1)</span>. This project implements the
+ singlestep functionality by two consecutive breakpoints
+ (<span class="instruction">hlt</span> instructions to be specific):
+ The first two instruction addresses of the W32 functions are called
+ <span class="productname">slot #1</span> and
+ <span class="productname">slot #2</span>, the length of the first
+ function instruction has to be analyzed to get the right address of
+ <span class="productname">slot #2</span>. When the first breakpoint is
+ caught it is necessary to patch the original instruction back and also
+ patch another breakpoint in place of
+ <span class="productname">slot #2</span>.
+ During the <span class="productname">slot #2</span> breakpoint
+ invocation the operation will be reverted - the breakpoint will be put
+ to <span class="productname">slot #1</span> again and the instruction
+ of <span class="productname">slot #2</span> will be restored to be able
+ to continue the execution of the function.</p>
+
+ <p>W32 function will finish in its specific
+ <a href="#calltype">cdecl/stdcall/fastcall call type</a>, the control
+ will return to the UNIX jump table relay which will debug dump the
+ return value and it will finally pass the control back to the UNIX
+ caller in the standard UNIX
+ <a href="#calltype_cdecl">cdecl call type</a>.</p>
+
+ @{[ doc_img 'fig/functype_patched_pass_fromunix',
+ 'Function Type: <span class="constant">pass</span> from UNIX Code' ]}
+
+ <a name="functype_pass_fromw32"><h4>Pass from W32 Code</h4></a>
+
+ <p>This function type is similiar to the
+ <a href="#functype_pass_fromunix">previous one</a> with the exception
+ of more complicated entry point. Unfortunately W32 libraries call their
+ own functions directly, using the <span class="instruction">call</span>
+ instructions without any patchable jump table. Even the
+ <span class="instruction">call</span> argument itself cannot be patched
+ according to the relocation table record as such library intra-call
+ instruction has no relocation due to its relative argument offset on
+ <span class="constant">i386</span>. This time the double-breakpoint
+ mechanism <a href="#functype_pass_fromunix">described above</a> gets
+ handy since it will catch the entry point when the function gets
+ called. <span class="constant">SIGSEGV</span> handler gets invoked by
+ the <span class="instruction">hlt</span> instruction and it will
+ redirect the control to the jump table relay function to debug dump the
+ function entry arguments (it has no other uses in this call type).</p>
+
+ <p>When the relay needs to call the original function it will reach
+ exactly the same breakpoint instruction as during the recent
+ <span class="constant">SIGSEGV</span> handling redirecting to this
+ calling relay. But this time the
+ <span class="constant">through_w32_func</span> field of this function
+ record will be set to to prevent repeated redirection and to pass the
+ control through the breakpoint mangle instead this time.</p>
+
+ <p>Returning is not much interesting as the first
+ <span class="constant">SIGSEGV</span> handler did a straight jump
+ for the redirection purposes without any needed consequent
+ handling.</p>
+
+ <p>The jump table relay used for the callers from W32 code is
+ a different one than the relay being used for the callers
+ <a href="#functype_pass_fromunix">from UNIX code</a>. UNIX code always
+ uses relay with external <a href="#calltype_cdecl">cdecl call type</a>
+ but in this case a relay with the appropriate
+ <a href="#calltype">cdecl/stdcall/fastcall call type</a> is used.</p>
+
+ @{[ doc_img 'fig/functype_patched_pass_fromw32',
+ 'Function Type: <span class="constant">pass</span> from W32 Code' ]}
+
+ @{[ vskip() ]}
+
+ <table border="1" align="center">
+ <tr><td><span class="fname">captivesym</span> keyword</td><td>pass</td></tr>
+ <tr><td>Native code function name </td><td>(no implementation)</td></tr>
+ <tr><td>W32 traced code from UNIX function name </td><td>FUNCNAME</td></tr>
+ <tr><td>W32 traced code from W32 function name </td><td>FUNCNAME_cdecl/_stdcall/_fastcall</td></tr>
+ <tr><td>Entry/exit debug tracing from UNIX code </td><td>yes</td></tr>
+ <tr><td>Entry/exit debug tracing from W32 code </td><td>yes</td></tr>
+ <caption>Function Type <span class="constant">pass</span> Characteristics</caption>
+ </table>
+
+ <a name="functype_wrap"><h3>Wrap of the Original "ntoskrnl.exe" Function</h3></a>
+
+ <a name="functype_wrap_fromunix"><h4>Wrapping of Call from UNIX Code</h4></a>
+
+ <p>The code control flow has no special hardcore features since it is
+ very similiar to <a href="#functype_pass_fromunix">the direct pass to
+ W32 function from UNIX code</a>. All the wrapping is done in the
+ standard UNIX <a href="#calltype_cdecl">cdecl call type</a> manner.
+ Jump table debug dumping relays are provided twice - the
+ "outer" one to trace the parameters from the function caller
+ and the "inner" one to trace the call from the wrapper to the
+ original W32 code. The "inner" relay also calls the W32 code
+ with the appropriate <a href="#calltype">cdecl/stdcall/fastcall call
+ type</a>.</p>
+
+ @{[ doc_img 'fig/functype_patched_wrap_fromunix',
+ 'Function Type: <span class="constant">wrap</span> from UNIX Code' ]}
+
+ <a name="functype_wrap_fromw32"><h4>Wrapping of Call from W32 Code</h4></a>
+
+ <p>This scheme is a combination of the
+ <a href="#functype_wrap_fromunix">previous wrap of a call from
+ UNIX code</a> and the <a href="#functype_pass_fromw32">direct pass from
+ the W32 code</a>. The control is caught and redirected by
+ <span class="constant">SIGSEGV</span> handler from the breakpoint
+ placed at the entry to the original W32 function code. The second entry
+ to the original W32 function with the
+ <span class="constant">through_w32_func</span> field of this function
+ description already set is done from the "inner" jump table
+ relay with the appropriate
+ <a href="#calltype">cdecl/stdcall/fastcall call type</a>.</p>
+
+ @{[ doc_img 'fig/functype_patched_wrap_fromw32',
+ 'Function Type: <span class="constant">wrap</span> from W32 Code' ]}
+
+ @{[ vskip() ]}
+
+ <p>Some functions can be <a href="#functype_pass">passed to the original
+ code</a> but they need their parameters to be checked/prepared.
+ Currently, such wrapping is only needed for the
+ <span class="function">ExAllocateFromPagedLookasideList()</span> function
+ where it is required due to <a href="#init_ntoskrnl">missing execution of
+ <span class="fname">ntoskrnl.exe</span> initialization execution</a>,
+ which would otherwise properly initialize some internal data structures.
+ In this case the wrapping code detects passing of an uninitialized
+ parameter and will search through the whole
+ <span class="fname">ntoskrnl.exe</span> code body at runtime to find the
+ proper initialization routine containing the correct initialization
+ parameters. Passed addresses of static structures must be differentiated
+ as each of them usually has different initialization parameters. It is
+ proactive to not to have fixed parameters array as these parameters may
+ differ across different <span class="fname">ntoskrnl.exe</span>
+ versions.</p>
+
+ <table border="1" align="center">
+ <tr><td><span class="fname">captivesym</span> keyword</td><td>wrap</td></tr>
+ <tr><td>Native UNIX wrapping code function name </td><td>FUNCNAME_wrap</td></tr>
+ <tr><td>W32 traced wraping code from UNIX func. name </td><td>FUNCNAME</td></tr>
+ <tr><td>W32 traced wrapping code from W32 func. name </td><td>FUNCNAME_cdecl/_stdcall/...</td></tr>
+ <tr><td>W32 traced original code function name </td><td>FUNCNAME_orig</td></tr>
+ <tr><td>Entry/exit debug tracing from UNIX code </td><td>yes</td></tr>
+ <tr><td>Entry/exit debug tracing from W32 code </td><td>yes</td></tr>
+ <caption>Function Type <span class="constant">wrap</span> Characteristics</caption>
+ </table>
+
+ <a name="functype_native"><h3>Native Implementation</h3></a>
+
+ <h4>Native Implementation Called from UNIX Code</h4>
+
+ <p>This is the simplest case of a function call as it is fully
+ handled only by the compiler and/or linker.</p>
+
+ <p>In this case though, no debug dumping call relay is provided - such
+ relay would need to rename the implementations of native functions to
+ prevent its automatic linking with the caller code. This renaming would
+ not be possible to do by simple <span class="constant">#define</span>
+ since it would also rename any calling statements of such function in
+ the same C sources. One of the possibilities to solve would be to
+ utilize <span class="dashdash">--redefine-sym</span> feature of the
+ <span class="productname">objcopy(1)</span> utility. On the other hand
+ there is not much need to catch/debug such calls as both the caller and
+ the callee are provided with full source file debug information for the
+ debugger. Also the callee usually debug dumps its entry/exit parameters
+ by custom debug dumps in the
+ <a href="#functype_native_reactos"><span class="productname">ReactOS</span> implementations</a>.
+
+ @{[ doc_img 'fig/functype_native_fromunix',
+ 'Function Type: <span class="constant">native</span> from UNIX Code' ]}
+
+ <a name="functype_native_fromw32"><h4>Native Implementation of
+ "unpatched" Library Function Called from W32 Code</h4></a>
+
+ @{[ doc_img 'fig/functype_unpatched_native_fromw32',
+ 'Function Type: <span class="constant">native</span> of <span class="constant">unpatched</span> from W32 Code' ]}
+
+ <p>Here comes the differentiation if the project deals either with
+ a <span class="constant">patched</span> or an
+ <span class="constant">unpatched</span> version of the library
+ (<span class="constant">patched</span> is a loaded W32 binary
+ library while <span class="constant">unpatched</span> library is
+ completely provided by this project with no use of the library's
+ original W32 binary file). As the project adjusts the exported symbol
+ address during the patching operation, in some cases the
+ <span class="constant">patched</span> library call may be handled
+ simply as <span class="constant">unpatched</span> library call even for
+ the <span class="constant">patched</span> libraries. Fortunately the
+ distinction is not much important as the project is prepared to
+ properly handle both cases.</p>
+
+ <p>The W32 caller which imported the symbol will be pointed right to
+ the relaying function. The debug dumping relay will be called from W32
+ code with the appropriate
+ <a href="#calltype">cdecl/stdcall/fastcall call type</a> while the
+ relay will call the implementation of the native function in the
+ standard UNIX <a href="#calltype_cdecl">cdecl call type</a> manner.</p>
+
+ <h4>Native Implementation of "patched" Library Function Called from W32 Code</h4>
+
+ @{[ doc_img 'fig/functype_patched_native_fromw32',
+ 'Function Type: <span class="constant">native</span> of <span class="constant">patched</span> from W32 Code' ]}
+
+ <p>The calling scheme is similiar to the
+ <a href="#functype_native_fromw32">previous call of
+ <span class="constant">unpatched</span> library function from W32
+ code</a> but the call control is redirected from the entry point of the
+ original W32 binary implementation by the breakpoint and its
+ <span class="constant">SIGSEGV</span> handler as in
+ <a href="#functype_pass_fromw32">the case of passing control from W32
+ call</a>.</p>
+
+ <p>The original W32 function implementation located in the original
+ loaded binary file is never executed but its entry point needs to be
+ trapped by the breakpoint to be able to catch the function calls within
+ the library.</p>
+
+ @{[ vskip() ]}
+
+ <p>In all cases the final function implementation is a standard UNIX
+ code compiled from C sources with full debug information available
+ for the debugger. Fortunately all such functions do not need to be coded
+ from scratch for this project since there already exist $freespeech
+ $ReactOS and $Wine projects and their code can be used instead.</p>
+
+ <p>$Wine project is listed mostly for a completeness as almost no
+ code was suitable for reuse as it implements W32 user space while this
+ project is running pure W32 kernel space environment (in $gnulinux user
+ space!).</p>
+
+ <a name="functype_native_reactos"><h4>Native Implementation
+ - <span class="productname">ReactOS</span></h4></a>
+
+ <p>Some functions are already implemented in the $ReactOS
+ project and they can be used as they are. Although it would be
+ possible to <a href="#functype_pass">pass some function calls to the
+ original code</a> it is more handy to provide native implementation as
+ there is better control of the data handling during debugging sessions
+ due to the provided debugging symbols.</p>
+
+ <p>Such functions can be found in
+ <span class="fname">src/libcaptive/reactos/</span> subdirectory.
+ Some functions had to be adjusted for this project
+ - these modifications are compiled conditionally, depending on the
+ <span class="constant">LIBCAPTIVE</span> symbol existence.</p>
+
+ <p>Later stages of this project reached the level where
+ $ReactOS is yet too immature and the needed functions are usually
+ written just with the sad body:</p>
+
+ <blockquote class="command">
+ <p>UNIMPLEMENTED;</p>
+ </blockquote>
+
+ <p>Functions that were not possible to
+ <a href="#functype_pass">pass</a> were reimplemented by this project
+ and placed in the project's implementation directories
+ <a href="#reactos_nocare">instead of extending</a> $ReactOS code.</p>
+
+ <a name="functype_native_wine"><h4>Native Implementation -- <span class="productname">Wine</span></h4></a>
+
+ <p>Even though $Wine only implements the
+ <span class="productname">Microsoft Windows NT</span> user space, there
+ still are some common functions which could be copied from the $Wine
+ project.</p>
+
+ <a name="functype_native_libcaptive"><h4>Native Implementation - Project Specific</h4></a>
+
+ <p>As the last resort it was necessary to provide completely own
+ implementation of some API functions such as PC hardware dependent
+ parts or memory management functions.</p>
+
+ @{[ vskip() ]}
+
+ <table border="1" align="center">
+ <tr><td><span class="fname">captivesym</span> keyword</td><td>(none; just the symbol name)</td></tr>
+ <tr><td>Native code function name </td><td>FUNCTIONNAME</td></tr>
+ <tr><td>Native traced code from W32 code func. name </td><td>FUNCTIONNAME_cdecl/_std...</td></tr>
+ <tr><td>Entry/exit debug tracing from UNIX code </td><td>no</td></tr>
+ <tr><td>Entry/exit debug tracing from W32 code </td><td>yes</td></tr>
+ <caption>Function Type <span class="constant">native</span> Characteristics</caption>
+ </table>
+
+ <a name="functype_undef"><h3>Undefined Function</h3></a>
+
+ <p>Functions not defined by any of the previous function types cannot be
+ called by any W32 code including the code of the library implementing
+ such function. All functions of <span class="constant">patch</span>ed
+ libraries not listed in the <span class="fname">captivesym</span> exports
+ file are automatically set to be trapped as fatal program execution
+ errors.</p>
+
+ <p>It is not necessary to list the symbols as
+ <span class="constant">undef</span> as long as you are just loading the
+ W32 <span class="constant">PE-32</span> code and the symbols belong to
+ <span class="constant">patch</span>ed library. On the other hand if you
+ are loading W32 <span class="fname">.so</span> code or if such symbol is
+ a part of <span class="constant">unpatched</span> library (and thus
+ being completely provided by the project) you need to list such symbol as
+ <span class="constant">undef</span> type to prevent unresolved symbol
+ reference.</p>
+
+ <table border="1" align="center">
+ <tr><td><span class="fname">captivesym</span> keyword</td><td>undef</td></tr>
+ <tr><td>Native code function name </td><td>(no implementation)</td></tr>
+ <tr><td>Native traced code function name </td><td>FUNCTIONNAME_cdecl/_stdcall/_fastcall</td></tr>
+ <tr><td>Debug tracing message from UNIX code </td><td>yes</td></tr>
+ <tr><td>Debug tracing message from W32 code </td><td>yes</td></tr>
+ <caption>Function Type <span class="constant">undef</span> Characteristics</caption>
+ </table>
+
+
+ <a name="calltype"><h2>API Function Calling Conventions</h2></a>
+
+ <p>Standard UNIX code compiled by GCC (GNU C Compiler) running on host
+ $gnulinux always uses <a href="#calltype_cdecl">cdecl</a> ABI (Application
+ Binary Interface) calling convention. This calling convention is also the
+ default declaration type of UNIX functions.</p>
+
+ <p>W32 uses three different calling conventions in its ABI. They are all
+ described in the
+ <a href="http://msdn.microsoft.com/library/default.asp?url=/library/en-us/vclang/html/_core_argument_passing_and_naming_conventions.asp"><span class="productname">Microsoft</span> documentation</a>.
+ There is always necessary to have the proper function declaration
+ (prototype) in the caller scope to prevent all sorts of unexpected
+ crashes.</p>
+
+ <p>Unfortunately some non-matching combinations of calling conventions
+ result in hard to debug bugs: the caller gets back an unexpected stack
+ pointer from the callee and upon return it will restore registers from the
+ wrong stack pointer place. Since the caller will finally reclaim its stack
+ frame from its (uncorrupted) <span class="constant">EBP</span> stack frame
+ pointer the caller will return to the caller of the caller correctly. Just
+ the registers remain corrupted causing crashes of completely unrelated code
+ executed far, far away...</p>
+
+ <p><span class="constant">EDI</span>, <span class="constant">ESI</span> and
+ <span class="constant">EBX</span> registers are always saved on the stack.
+ They are stored on the stack in this particular order from bottom to top
+ addresses (using the <span class="instruction">push EBX</span>,
+ <span class="instruction">push ESI</span>,
+ <span class="instruction">push EDI</span> sequence). Fortunately $gnulinux
+ GCC has the same register saving behaviour. If some register corruption
+ occurs the calling type presented between the caller and callee should be
+ checked.</p>
+
+ <a name="calltype_cdecl"><h3>W32 Calling Convention "cdecl"</h3></a>
+
+ <p>The only calling convention in the UNIX world. The default one for all
+ the compilers. All the arguments are passed on the stack, no arguments
+ are cleaned by the callee. Possible inconsistencies in the number of
+ function arguments with the function prototype used by the caller is
+ harmless. Variable arguments lists can be passed by this convention.</p>
+
+ @{[ doc_img 'fig/calltype_cdecl',
+ 'W32 Calling Convention <span class="constant">cdecl</span> Scheme' ]}
+
+ <table border="1" align="center">
+ <tr><td>Arguments freed by </td><td>caller</td></tr>
+ <tr><td>Arguments on the stack </td><td>#0 ... #(n-1)</td></tr>
+ <tr><td>Arguments in the registers </td><td>none</td></tr>
+ <tr><td>GCC attribute </td><td><span class="command">__attribute__((__cdecl__))</span> (default)</td></tr>
+ <caption>Calling Convention <span class="constant">cdecl</span> Characteristics</caption>
+ </table>
+
+ <h3>W32 Calling Convention "stdcall"</h3>
+
+ @{[ doc_img 'fig/calltype_stdcall',
+ 'W32 Calling Convention <span class="constant">stdcall</span> Scheme' ]}
+
+ <p>Convention never used in the UNIX world. It needs to be specified for
+ W32 compilers. All the arguments are passed on the stack, all the
+ arguments are cleaned by the callee. Possible inconsistencies in the
+ number of function arguments with the function prototype used by the
+ caller will result in fatal crash. Variable arguments lists cannot be
+ passed by this convention - use <a href="#calltype_cdecl">cdecl</a>
+ instead.</p>
+
+ <table border="1" align="center">
+ <tr><td>Arguments freed by </td><td>callee</td></tr>
+ <tr><td>Arguments on the stack </td><td>#0 ... #(n-1)</td></tr>
+ <tr><td>Arguments in the registers </td><td>none</td></tr>
+ <tr><td>GCC attribute </td><td><span class="command">__attribute__((__stdcall__))</span></td></tr>
+ <caption>Calling Convention <span class="constant">stdcall</span> Characteristics</caption>
+ </table>
+
+ <h3>W32 Calling Convention "fastcall"</h3>
+
+ <p>Convention never used in the UNIX world. It needs to be specified for
+ W32 compilers. Convention used in the W32 world for its low calling
+ overhead. All but the first two arguments are passed on the stack, such
+ arguments are cleaned by the callee. First two arguments are passed in
+ the registers <span class="constant">ECX</span> and
+ <span class="constant">EDX</span> respectively. Possible inconsistencies
+ in the number of function arguments with the function prototype used by
+ the caller will result in fatal crash. Variable arguments lists cannot be
+ passed by this convention - use <a href="#calltype_cdecl">cdecl</a>
+ instead.</p>
+
+ <p>GCC (GNU C Compiler) native support for this calling convention
+ is pretty fresh and it is currently present only in the recent CVS
+ versions since 21st December of 2002 which should get released as GCC
+ version 3.4. This project solved the unsupported calling convention by
+ declaration of arguments passed in registers by
+ <span class="command">__attribute__((__regparm__(3)))</span>.
+ W32 passes the arguments in registers in the order
+ <span class="constant">ECX</span>, <span class="constant">EDX</span> but
+ GCC passes them in registers <span class="constant">EAX</span>,
+ <span class="constant">EDX</span>, <span class="constant">ECX</span>.
+ This incompatibility is compensated at C source level in the
+ <a href="#functype">relaying code</a> generated by
+ <span class="fname">captivesym</span> relay generator.</p>
+
+ @{[ doc_img 'fig/calltype_fastcall',
+ 'W32 Calling Convention <span class="constant">fastcall</span> Scheme' ]}
+
+ <table border="1" align="center">
+ <tr><td>Arguments freed by </td><td>callee</td></tr>
+ <tr><td>Arguments on the stack </td><td>#2 ... #(n-1)</td></tr>
+ <tr><td>Arguments in the registers </td><td><span class="constant">ECX</span>=#0,
+ <span class="constant">EDX</span>=#1</td></tr>
+ <tr><td>GCC ≥3.4 attribute </td><td><span class="command">__attribute__((__fastcall__))</span></td></tr>
+ <tr><td>GCC <3.4 attr. emulation</td><td><span class="command">__attribute__((__stdcall__))</span></td></tr>
+ <tr><td> </td><td><span class="command">__attribute__((__regparm__(3) /* EAX,EDX,ECX */))</span></td></tr>
+ <caption>Calling Convention <span class="constant">fastcall</span> Characteristics</caption>
+ </table>
+
+ <a name="synchronous"><h2>Multithreading and Multiple Processors</h2></a>
+
+ <p>W32 platform stands on its thorough architecture parallelism. It
+ must lock all its objects to maintain coherence in presence of
+ multithreading and multiple processors. Since the author of this project
+ considers any parallel execution a serious obstacle for debugging the whole
+ project architecture was designed to prevent any undeterministic behaviour.
+ Therefore this projects always emulates uniprocessor
+ <span class="productname">Microsoft Windows NT</span> kernel
+ (<span class="constant">KeNumberProcessors</span> symbol is always 1),
+ everything runs in the single initial thread/process and all the filesystem
+ operations are performed as synchronous
+ ("synchronous" by flags
+ <span class="constant">FILE_SYNCHRONOUS_IO_ALERT</span>,
+ <span class="constant">FO_SYNCHRONOUS_IO</span>,
+ <span class="constant">IRP_SYNCHRONOUS_API</span>,
+ <span class="constant">IRP_SYNCHRONOUS_PAGING_IO</span>,
+ forced <span class="constant">TRUE</span> result of
+ <span class="function">IoIsOperationSynchronous()</span>
+ etc.).
+ <span class="constant">STATUS_PENDING</span> result code indicating that
+ request should be completed in the next callback of the driver is
+ considered <a href="#paranoia">fatal</a> as it should not happen for the
+ requested synchronous <span class="constant">IRP</span>s (I/O Request
+ Packets). Since there is a possibility some filesystem would require
+ a real W32 parallel thread all the code that would be hit by W32
+ multithreading capability is marked by
+ <span class="constant">TODO:thread</span> comment for a possible
+ future extension.</p>
+
+ <p>Multiple processors (SMP) support will never need to be implemented
+ since uniprocessor W32 kernels apparently run the filesystem driver modules
+ fine. As this project implements only the uniprocessor W32 kernel all the
+ processor locking functions and structures such as
+ <span class="constant">KSPIN_LOCK</span> etc. can be safely implemented as
+ no-operations.</p>
+
+ <p>Asynchronous callbacks registered for
+ <span class="constant">IO_WORKITEM</span>s are passed as GLib idle
+ functions by <span class="function">g_idle_add_full()</span>. Although they
+ will probably never be executed during non-interactive project's batch
+ executions it is the responsibility of W32 driver implementation to
+ complete all the pending tasks before its W32 shutdown. Such W32 shutdown
+ is done during cleanup of the project's execution by
+ <span class="function">captive_shutdown()</span>.</p>
+
+ <a name="paranoia"><h2>Paranoia Checks</h2></a>
+
+ <p>A general approach of software projects development is to implement
+ many internal sanity checks during the development stage but to produce the
+ most optimized final release product without those debugging checks.</p>
+
+ <p>Facilities for these practices can be seen in the standard
+ C include files for example as function
+ <span class="function">assert()</span> which gets disabled by the
+ <span class="constant">NDEBUG</span> symbol used during the final optimized
+ executable compilation. This project uses Gnome GLib messaging subsystem
+ offering sanity checks discarded by symbols
+ <span class="constant">G_DISABLE_ASSERT</span> and
+ <span class="constant">G_DISABLE_CHECKS</span>.
+ <span class="productname">Microsoft</span> also produces two versions of
+ its products - regular customers use the "free build" (also
+ called "retail") while the programmers should develop their code
+ on the "checked build" product releases.</p>
+
+ <p>As this project will always run unknown binary code of proprietary W32
+ filesystem drivers, the code can never be trusted. Such code even runs in
+ the same unprotected address space as its controlling UNIX code. Since
+ there is not enough documentation for the W32 components of the system and
+ also such documentation is usually misleading it can never be considered as
+ 100% emulation. Even in the final releases all the sanity checks
+ implemented in this project should remain active as all the project's code
+ always interacts with unknown and untrusted W32 binaries.</p>
+
+ <p><span class="productname">Microsoft Windows NT</span> code is written in
+ a foolproof style as it accepts even invalid input values, and which
+ it usually corrects. This makes long-term debugging a pain as it hides
+ sources of problems. "Checked build" releases were probably
+ designed to fix this flaw by strict consistency checks but it did not reach
+ its goals as such checks are usually missing in the code.</p>
+
+ <p>This project has strict consistency checks across all the code to make
+ the debugging phase easy enough. Failed sanity check is not always
+ a bug - sometimes it just means the real W32 binary code is more
+ benevolent than it could be expected according to the documentation and
+ such sanity check gets removed for the next version build. In other cases
+ the failed sanity checks mean the execution path for some unexpected
+ arguments combination was not yet implemented by this project. I may also
+ mean a bug, of course...</p>
+
+ <p>Last but not least - never miss a possible sanity check as its
+ later removal is in an order of magnitude cheaper than an uncaught
+ invalid assumption. Failed assertion is not always a bug although it
+ has to be fixed, of course.</p>
+
+ <a name="client_interface"><h2>Client Filesystem Interface</h2></a>
+
+ <p>While this project successfuly communicates with the W32 filesystem
+ driver (considered as the lower layer) it must also somehow offer its open
+ filesystem interface service to some real client software (upper layer).
+ This project offers its own custom filesystem operations interface of <span
+ class="constant">libcaptive</span> library based on GLib
+ <span class="constant">GObject</span> OO system. Interface prototypes are
+ specified in the project's <span class="fname">client-*.h</span>
+ include files.</p>
+
+ <p>The filesystem service can be offered in several ways:</p>
+
+ <ul>
+ <li>
+ <p>One possibility would be to write
+ <a name="client_interface_customapp">a custom client application</a>
+ for this project such as file manager or a shell. Although it
+ would implement the most appropriate user interface to the set of
+ functions offered by this project (and W32 filesystem API) it has the
+ disadvantage of special client software. Appropriate client is provided
+ by this project as:
+ <span class="fname">src/client/cmdline/cmdline-captive</span></p>
+ </li>
+ <li>
+ <p>The real UNIX OS filesystem implementation must be completely
+ implemented inside the hosting OS kernel. This requires special coding
+ methods with limited availability of coding features and libraries.
+ Also it would give the full system control to the untrusted W32
+ filesystem driver code with possibly fatal consequences of yet
+ unhandled W32 emulation code paths. It would benefit from the best
+ execution performance but this solution was never considered a real
+ possibility.</p>
+ </li>
+ <li>
+ <p>The common approach
+ <a name="offered_NFS">of filesystem implementations</a>
+ outside UNIX OS kernel were custom NFS servers usually running on the
+ same machine as the NFS-connected client as such NFS server is usually
+ an ordinary UNIX user space process. It would be possible to implement
+ this project as a custom NFS server but the NFS protocol itself
+ has a lot of fundamental flaws and complicated code for backward
+ compatibility.</p>
+ </li>
+ <li>
+ <p>Currently there is already implemented
+ <a name="offered_gnomevfs"><a href="#offered_gnomevfs_todo">Gnome-VFS interface</a></a>
+ to the custom filesystem interface of this project's library <span
+ class="constant">libcaptive</span>.
+ The $GnomeVFSmodule can be used by a Gnome-VFS aware client (such
+ as <span class="fname">gnome-vfs/tests/test-shell</span>).</p>
+
+ <FIXME:lufs-gvfs>
+ <p>The <span class="productname">Gnome-VFS-module</span> can be further
+ utilized by the <span class="productname">UserVFS</span>
+ \bookcitation{UserVFS-2.0} software ported to provide local <span
+ class="productname">Coda</span> \bookcitation{Coda} network filesystem
+ server implementation similar to the <a href="#offered_NFS">NFS
+ server</a> solution but with much more acceptable network protocol ---
+ more about this actual scheme can be found in \link{architecture}{the
+ project architecture description}.</p>
+ </li>
+ <li>
+ <FIXME:LUFS>
+ <p>Direct interface for the Host-OS kernel would be provided
+ by the
+ \label{fuse_interface}
+ <span class="productname">FUSE</span> \bookcitation{FUSE} project \link{offered_FUSE}{described
+ later in this document}. This interface is currently not yet implemented.
+ Although it would be much more straightforward than
+ <a href="#offered_gnomevfs">Gnome-VFS interface</a> described above,
+ its biggest disadvantage would be the requirement to replace/update
+ the stock distributions kernel package as it usually does not
+ have the <span class="productname">FUSE</span> \bookcitation{FUSE} filesystem support while it already supports
+ the <span class="productname">Coda</span> \bookcitation{Coda} interface, which is sufficient for the
+ ported <span class="productname">UserVFS</span> \bookcitation{UserVFS-2.0} interface.</p>
+ </li>
+ </ul>
+
+ <h2>3rd Party Projects Bugfixes</h2>
+
+ <p>Implementation of this project required certain bugfixes to 3rd party
+ software packages:</p>
+
+ <h3>GNU Libtool, A Generic Library Support Script</h3>
+
+ <p><span class="productname"><a href="http://www.gnu.org/software/libtool/">libtool</a></span>:
+ Handle duplicate object file names when performing piecewise archive
+ linking by renaming object files when needed.</p>
+
+ <h3>dosfstools, MS-DOS FAT Filesystems Support on Linux</h3>
+
+ <p><span class="productname"><a href="ftp://ftp.uni-erlangen.de/pub/Linux/LOCAL/dosfstools/">dosfstools</a></span>:
+ Prevent generation of <span class="constant">FAT-32</span> filesystems
+ not supported by the (buggy?) W32 platform
+ <span class="fname">fastfat.sys</span> implementation.</p>
+
+ <h3>ext2fsd, Ext2 File System Driver</h3>
+
+ <p><span class="productname"><a href="http://sys.xiloo.com/projects/projects.htm#ext2fsd">Ext2fsd</a></span>:
+ Many filesystem corruption fixes, missing filesystem unregistration
+ etc.</p>
+
+
+<h1>Futher Development</h1>
+
+ <p>All the W32 filesystem operations of <span class="fname">cdfs.sys</span>,
+ <span class="fname">fastfat.sys</span>
+ and
+ <span class="fname">ext2fsd.sys</span> can be successfuly executed.
+ The further development tasks include:</p>
+
+ <ul>
+ <li>
+ <p>The primary goal is to reach <span class="productname">NTFS</span>
+ filesystem (<span class="fname">ntfs.sys</span>) compatibility.
+ A lot of imported symbols is missing although it is expected most of
+ them can be just safely passed for execution in the original
+ <span class="fname">ntoskrnl.exe</span>.</p>
+ </li>
+ <li>
+ <p>There may still be valid code paths where some emulated W32 kernel
+ functionality and symbols remain unimplemented as these code paths were
+ just not hit during testing. The proper way would be to check all the
+ possibilities of such code paths execution from the filesystem driver
+ code disassembly.</p>
+ </li>
+ <li>
+ <p>No unusual error codes are expected from the filesystem drivers and
+ any such return codes will abort the project's execution. For example
+ code <span class="constant">STATUS_NO_SUCH_FILE</span> is expected and
+ correctly recognized but
+ <span class="constant">STATUS_FILE_CORRUPT_ERROR</span> will stop driver
+ execution.</p>
+
+ <p><a name="exception_fatal">No exceptions in W32 code are allowed</a>
+ - any thrown exception will result in driver execution abortion (instead
+ of just returning some error code as in the original W32 environment).</p>
+
+ <p>These issues should cease to be a problem after deployment of
+ sandbox wrapper which will restart the filesystem driver after any
+ unexpected error.</p>
+ </li>
+ <li>
+ <p><a name="todo_sandbox">Completion and activation of the sandbox
+ wrapper.</a> <span class="fname">src/libcaptive/sandbox/</span> sources
+ currently implement the base of both the client and the server sides of
+ CORBA interface to separate the client calling filesystem operations from
+ the W32 filesystem driver itself. Although CORBA usually makes sense for
+ crossmachine network interconnections here it gets a role of
+ inter-process interface between the regular client process and the
+ <span class="constant">chroot</span>ed/unprivileged/<span class="constant">ulimit</span>ed
+ environment of the W32 emulation address space.</p>
+
+ <p>Any W32 binary file must be always considered untrusted and therefore
+ it is needed to be sandboxed and accessible only via the CORBA interface.
+ Furthermore it is needed for clean implementation of $GnomeVFSmodule as
+ this project always handles <a href="#mounted_one">exactly one mounted
+ filesystem</a> but $GnomeVFSmodule interface expects unlimited number of
+ mounts in the scope of one process.</p>
+ </li>
+ <li>
+ <p>Project offers
+ <a name="offered_gnomevfs_todo">the filesystem access as its custom UNIX API</a>
+ (<span class="fname">captive/client-*.h</span>). This API is currently
+ offered in the scope of $GnomeVFSmodule interface as a filter applied to
+ the filesystem device (or filesystem image file).
+ As $GnomeVFS has no officially supported method of generic $gnulinux
+ kernel filesystem access it may be better to provide
+ <FIXME:LUFS><a name="offered_FUSE">an interface</a> for <span
+ class="productname">FUSE</span> \bookcitation{FUSE} instead.</p>
+
+ <p>To get transparent access to W32 filesystems from legacy
+ (=non <span class="productname">Gnome-VFS-2.0</span> aware) applications it is possible to use a draft
+ port \bookcitation{UserVFS-2.0} of the original <span class="productname">UserVFS</span>
+ \bookcitation{UserVFS} to <span class="productname">Gnome-VFS-2.0</span> interface.
+ It is also possible to use the test utilities of <span class="productname">Gnome-VFS-2.0</span> \bookcitation{GnomeVFS} package.</p>
+ </li>
+ <li>
+ <p>Implementation of interface to this project by
+ <span class="productname"><a href="http://surprise.sourceforge.net/">Partition Surprise</a></span>
+ partition manager. Although there currently exists
+ <span class="productname"><a href="http://mlf.linux.rulez.org/mlf/ezaz/ntfsresize.html">ntfsresize</a></span>
+ it is a data structures reverse engineered solution which may have
+ problems on various hard drives. <span class="productname">Partition
+ Surprise</span> project would be able to resize the disk safely by using
+ just the original W32 filesystem driver file although with some
+ performance hit.</p>
+ </li>
+ </ul>
+
+
+<h1>Related Projects</h1>
+
+ <p>The usual solution for file exchange between $freespeech operating systems
+ and <span class="productname">Microsoft Windows NT</span> is to use
+ <span class="productname">FAT32</span> (<span class="productname">vfat</span>
+ called in $gnulinux) partition and swap the files over it. This method is not
+ very comfortable as you never have access to all the files of the other
+ operating system.</p>
+
+ <a name="LinuxNTFScompet"><h2>$LinuxNTFS</h2></a>
+
+ <p>Although this project takes a completely different approach and has
+ a different architecture, the final goal is the same as for this
+ project - reliable read-write <span class="productname">NTFS</span>
+ filesystem support. $LinuxNTFS goes the way of reverse engineering
+ filesystem data structures (and possibly
+ <span class="fname">ntfs.sys</span> itself). Unfortunately after many years
+ of its development it did not yet reach the state of reliable read-write
+ access although its read-only part is considered trustworthy.</p>
+
+ <p>Using $LinuxNTFS for read-only access to existing partition with
+ <span class="productname">Microsoft Windows NT</span> installation is
+ planned to be able to acquire existing <span class="fname">ntfs.sys</span>,
+ <span class="fname">ntoskrnl.exe</span> and possibly
+ <span class="fname">ksecdd.sys</span> (imported by
+ <span class="fname">ntfs.sys</span>) files from the user's
+ <span class="productname">NTFS</span> partition.</p>
+
+ <h2><span class="productname"><a href="http://www.cgsecurity.org/ntfs.html">NTPwd NTFS Driver</a></span></h2>
+
+ <p>DOS based <a href="http://www.gnu.org/licenses/gpl.html">GPL-2.0</a>
+ read-write NTFS driver. Filesystem structures are reverse engineered in the
+ way of <a href="#LinuxNTFScompet">Linux-NTFS Project</a>. As it is not very
+ actively maintained it reaches a lower level of
+ <span class="productname">NTFS</span> compatibility.</p>
+
+ <h2>Virtual Machine with <span class="productname">Microsoft Windows NT</span></h2>
+
+ <p>Original <span class="productname">Microsoft Windows NT</span>
+ operating system can be run inside a virtual machine running under
+ $gnulinux (or vice versa) and share the read-write disk partitions by using
+ a network file sharing through a virtual network card.</p>
+
+ <p>Although there will be full filesystem structures compatibility the
+ <span class="productname">NTFS</span> partition cannot be accessed with no
+ system installed (or with non-bootable crashed system to repair it this
+ way)
+ (Although this project requires the original
+ <span class="fname">ntfs.sys</span> it can obtained from the legal
+ <span class="productname">Microsoft Windows NT</span> CD.),
+ it will have substantial system resources requirement and you also need
+ a virtual machine software product such as commercial
+ <span class="productname"><a href="http://www.vmware.com/download/workstation.html">VMware Workstation</a></span>.</p>
+
+
+<h1>Conclusion</h1>
+
+ <p>The project established <a href="#existing_emulation">a new form</a>
+ of W32 emulation model suitable for existing proprietary binary W32 kernel
+ code (drivers) while being hosted in an open source operating system
+ (currently $gnulinux). Currently, only the subsystems required by W32
+ filesystem drivers are implemented but the project can be further extended
+ for compatibility with various hardware-related drivers such as W32 video
+ drivers, W32 disk interface drivers etc.</p>
+
+ <p>Some W32 kernel space subsystems were implemented for the first time as
+ $freespeech code as they are still missing in the only currently available
+ $freespeech W32 kernel implementation, $ReactOS. Some W32 kernel function
+ behaviour expected by the drivers had to be reverse engineered and documented
+ in this project's API documentation (not listed in this book) and/or in
+ its source files, because its description in the
+ <span class="productname">Microsoft</span> documentation is missing.</p>
+
+ <p>Author had to get familiar both with the W32 kernel API and also with the
+ W32 kernel code by the reverse engineering. This experience also covers the
+ first <span class="productname">Microsoft Windows</span> compatible code ever
+ written by the author - <span class="fname">hal.dll</span> (Hardware
+ Abstraction Layer) part of W32 kernel.</p>
+
+ <p>Certain UNIX implementation interfaces allow a regular, non-privileged
+ user of UNIX system to mount image files with any W32 filesystem supported by
+ this project. Such mount operation usually requires UNIX
+ <span class="constant">root</span> privileges to do so. On the other hand the
+ choice of supported filesystem types is very limited as only a few
+ filesystem types are supported for the W32 platform.</p>
+HERE
+
+
+My::Web->footer();