Home | History | Annotate | Download | only in rpcsvc
      1 /*
      2  * CDDL HEADER START
      3  *
      4  * The contents of this file are subject to the terms of the
      5  * Common Development and Distribution License (the "License").
      6  * You may not use this file except in compliance with the License.
      7  *
      8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
      9  * or http://www.opensolaris.org/os/licensing.
     10  * See the License for the specific language governing permissions
     11  * and limitations under the License.
     12  *
     13  * When distributing Covered Code, include this CDDL HEADER in each
     14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     15  * If applicable, add the following below this CDDL HEADER, with the
     16  * fields enclosed by brackets "[]" replaced with your own identifying
     17  * information: Portions Copyright [yyyy] [name of copyright owner]
     18  *
     19  * CDDL HEADER END
     20  */
     21 /*
     22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
     23  * Use is subject to license terms.
     24  */
     25 
     26 #if defined(USE_FOR_SNOOP)
     27 %#include "ds_nfs_com.h"
     28 %#include "ds_prot.h"
     29 #else
     30 #if defined(RPC_XDR) || defined(RPC_SVC) || defined(RPC_CLNT)
     31 %#include <nfs/ds.h>
     32 #endif
     33 #endif
     34 
     35 %#include <nfs/nfs41_fhtype.h>
     36 
     37 const NFS_FH4MAXDATA		= 26;
     38 
     39 %#include <nfs/ds_filehandle.h>
     40 
     41 /*
     42  *  Dot-x file for the data server control protocol.
     43  */
     44 
     45 /*
     46  * Control Protocol identifier
     47  */
     48 typedef uint64_t	ds_id;
     49 typedef uint64_t	mds_dataset_id;
     50 typedef uint64_t	ds_verifier;
     51 
     52 enum ds_status {
     53       	DS_OK		= 0,
     54       	DSERR_ACCESS,
     55 	DSERR_ATTR_NOTSUPP,
     56       	DSERR_BAD_COOKIE,
     57       	DSERR_BAD_FH,
     58 	DSERR_BAD_MDSSID,
     59       	DSERR_BAD_STATEID,
     60       	DSERR_EXPIRED,
     61       	DSERR_FHEXPIRED,
     62       	DSERR_GRACE,
     63       	DSERR_INVAL,
     64 	DSERR_NOENT,
     65       	DSERR_NOT_AUTH,
     66       	DSERR_NOSPC,
     67       	DSERR_NOTSUPP,
     68       	DSERR_OLD_STATEID,
     69 	DSERR_PNFS_NO_LAYOUT,
     70       	DSERR_RESOURCE,
     71       	DSERR_SERVERFAULT,
     72       	DSERR_STALE,
     73       	DSERR_STALE_CLIENTID,
     74 	DSERR_STALE_DSID,
     75       	DSERR_STALE_STATEID,
     76       	DSERR_TOOSMALL,
     77       	DSERR_WRONGSEC,
     78 	DSERR_XDR,
     79       	DSERR_ILLEGAL
     80 };
     81 
     82 /*
     83  * XXX: Note that the supported bits have yet to be defined
     84  * and the size field would of course correspond to the
     85  * file size.
     86  */
     87 struct ds_attr {
     88 	int		ds_attrmask;
     89 	uint64_t	ds_size;
     90 };
     91 
     92 struct identity {
     93 	ds_verifier	boot_verifier;
     94 	opaque		instance<MAXPATHLEN>;
     95 };
     96 
     97 /*
     98  * DS_EXIBI - Exchange Identity and Boot Instance
     99  *
    100  *  ds_ident  : An identiifier that the MDS can use to distinguish
    101  *             between data-server instances.
    102  */
    103 struct DS_EXIBIargs {
    104 	identity	ds_ident;
    105 };
    106 
    107 /*
    108  * ds_id     : The 'short-hand' identifier for the DS (MDS assigned)
    109  *
    110  * mds_id    : The 'short-hand' identifer for the MDS
    111  *
    112  * mds_boot_verifier: An identifier that the data-server can use to determine
    113  *		      if an MDS has rebooted.
    114  *
    115  * mds_lease_period: A hint to the data-server on the lease period
    116  * 	  	     currently in effect at the MDS.
    117  */
    118 struct DS_EXIBIresok {
    119       	ds_id		ds_id;
    120 	ds_id		mds_id;
    121       	ds_verifier	mds_boot_verifier;
    122       	uint32_t	mds_lease_period;
    123 };
    124 
    125 union DS_EXIBIres switch (ds_status status) {
    126 case DS_OK:
    127 	DS_EXIBIresok        res_ok;
    128 default:
    129 	void;
    130 };
    131 
    132 /*
    133  * DS_CHECKSTATEargs -
    134  *
    135  * The message from a DS that is asking for the presented file state
    136  * to be verified.
    137  *
    138  *   stateid:
    139  *
    140  *   The stateid the client presented for the I/O.
    141  *
    142  *   fh:
    143  *
    144  *   File handle that the DS received in the compound (via PUTFH)
    145  *   from the client when the client performed I/O.
    146  *
    147  *   co_owner:
    148  *
    149  *   client owner (MUST be same as MDS client owner)
    150  */
    151 struct DS_CHECKSTATEargs {
    152       	stateid4	stateid;
    153       	nfs_fh4		fh;
    154 	client_owner4	co_owner;
    155 };
    156 
    157 /*
    158  * ds_filestate -
    159  *
    160  * The reply to the DS from MDS that confirms the
    161  * validity of the presented state.
    162  *
    163  *   mds_clid:
    164  *
    165  *   The MDS client id that corresponds to the presented stateid.
    166  *
    167  *   layout:
    168  *
    169  *   The layout of the file.  This allows the DS
    170  *   to determine if the offset that the client is performing
    171  *   I/O to is valid based on the layout.
    172  *
    173  *   open_mode:
    174  *
    175  *   The effective open mode for the object + clientid, based on the
    176  *   mode open() mode and the export mode
    177  *
    178  */
    179 struct ds_filestate {
    180       	clientid4 mds_clid;
    181       	layout4   layout<>;
    182       	int       open_mode;
    183 };
    184 
    185 union DS_CHECKSTATEres switch (ds_status status) {
    186 case DS_OK:
    187       	ds_filestate    file_state;
    188 default:
    189       	void;
    190 };
    191 
    192 /*
    193  * DS_FMATPT -
    194  *
    195  * A control protocol message that is used to transport FMA
    196  * telemetry data to MDS.
    197  *
    198  * This is a placeholder for a post pNFS/Basic putback.
    199  */
    200 struct DS_FMATPTargs {
    201       	opaque      fma_msg<>;
    202 };
    203 
    204 
    205 struct DS_FMATPTres {
    206       	ds_status status;
    207 };
    208 
    209 /*
    210  * DS_MAP_MDS_DATASET_ID
    211  *
    212  * For a given MDS Dataset ID at the MDS return the root path.
    213  *
    214  */
    215 struct DS_MAP_MDS_DATASET_IDargs {
    216       	mds_dataset_id mds_dataset_id;
    217 };
    218 
    219 struct DS_MAP_MDS_DATASET_IDresok {
    220       	ds_status   status;
    221       	utf8string  pathname;
    222 };
    223 
    224 union DS_MAP_MDS_DATASET_IDres switch (ds_status status) {
    225 case DS_OK:
    226       	DS_MAP_MDS_DATASET_IDresok res_ok;
    227 default:
    228       	void;
    229 };
    230 
    231 /*
    232  * DS_MAP_MDSSID
    233  *
    234  * Return a mapping for a given MDS Storage ID.
    235  */
    236 struct DS_MAP_MDSSIDargs {
    237 	mds_sid	mma_sid;
    238 };
    239 
    240 enum storage_type {
    241 	ZFS = 1
    242 };
    243 
    244 union ds_guid switch (storage_type stor_type) {
    245 case ZFS:
    246 	opaque	zfsguid<>;
    247 default:
    248 	void;
    249 };
    250 
    251 /*
    252  * ds_guid_map -
    253  *
    254  *	The mapping between the local data server guid value and the id-value
    255  *	assigned by the MDS for mapping.
    256  *
    257  *	If the storage type is ZFS, encoded in the ds_guid will be
    258  *	a ds_zfsguid.  Other storage types may have their own guid
    259  *	definition.
    260  *
    261  *	Encoded in the mds_sid field will be an array of mds_sid_contents.
    262  */
    263 struct ds_guid_map {
    264 	ds_guid		ds_guid;
    265       	mds_sid		mds_sid_array<>;
    266 };
    267 
    268 struct DS_MAP_MDSSIDresok {
    269 	ds_guid_map	guid_map;
    270 };
    271 
    272 union DS_MAP_MDSSIDres switch (ds_status status) {
    273 case DS_OK:
    274       	DS_MAP_MDSSIDresok res_ok;
    275 default:
    276       	void;
    277 };
    278 
    279 /*
    280  * DS_RENEW -
    281  *
    282  * A message from the DS to MDS used to exchnage
    283  * boot instances. This can be used to indicate to
    284  * the MDS when a data server has rebooted, and
    285  * also to the data-server when the MDS has rebooted.
    286  *
    287  * The data-server should drop all state when it
    288  * detects that the MDS has rebooted.
    289  *
    290  * The MDS should drop any state that it believes
    291  * the data-server is holding.
    292  *
    293  */
    294 struct DS_RENEWargs {
    295       	ds_id		ds_id;
    296       	ds_verifier	ds_boottime;
    297 };
    298 
    299 union DS_RENEWres switch (ds_status status) {
    300 case DS_OK:
    301       	ds_verifier mds_boottime;
    302 default:
    303       	void;
    304 };
    305 
    306 /*
    307  * ds_zfsattr -
    308  *
    309  * Attribute that pertains to the zfs storage information/state etc.
    310  *
    311  *    attrname:
    312  *
    313  *    Name of attribute.
    314  *
    315  *    attrvalue:
    316  *
    317  *    Value of the attribute.
    318  *
    319  * Supported attributes:
    320  *
    321  *
    322  * +----------+------------+-----------------------------------------
    323  * | attrname | type       | Description
    324  * +----------+------------+-----------------------------------------
    325  * | state    | boolean    | current status of pool: 0=offline,
    326  * |          |            | 			     1=online
    327  * +----------+------------+-----------------------------------------
    328  * | size     | uint64_t   | bytes free on dataset.
    329  * +----------+------------+-----------------------------------------
    330  * | config   |  attrvalue: xxx - look at how zpool status
    331  * |          |   displays the pool
    332  * |          |  configuration.
    333  * +----------+------------+-----------------------------------------
    334  * | dataset  | utf8string | hostname:root_pool/root_data_set
    335  * +----------+------------+-----------------------------------------
    336  */
    337 struct ds_zfsattr {
    338       	utf8string  attrname;
    339       	opaque      attrvalue<>;
    340 };
    341 
    342 typedef uint32_t   ds_addruse;
    343 
    344 /*
    345  *  Intended usage for the addresses.
    346  */
    347 const NFS      = 0x00000001;
    348 const DSERV    = 0x00000002;
    349 
    350 /*
    351  * ds_addr -
    352  *
    353  * A structure that is used to specify an address and
    354  * its usage.
    355  *
    356  *    addr:
    357  *
    358  *    The specific address on the DS.
    359  *
    360  *    validuse:
    361  *
    362  *    Bitmap associating the netaddr defined in "addr"
    363  *    to the protocols that are valid for that interface.
    364  */
    365 struct ds_addr {
    366 	netaddr4	addr;
    367 	ds_addruse	validuse;
    368 };
    369 
    370 /*
    371  * ds_zfsguid -
    372  *
    373  *	The data server guid made up of the local zpool guid + dataset id
    374  */
    375 struct ds_zfsguid {
    376 	uint64_t	zpool_guid;
    377 	uint64_t	dataset_guid;
    378 };
    379 
    380 /*
    381  * ds_zfsinfo -
    382  *
    383  * Contains all the attributes that pertain to the specified ZFS storage
    384  * identifier.
    385  *
    386  *    guid_map:
    387  *
    388  *    Unique value identifying the pNFS dataset and the zpool it lives in.
    389  *    *Note: This has to be unique across all storage in our
    390  *           system.  Meaning that we may have to generate ids
    391  *           which take into account the poolid assigned by the  SPA
    392  *           and the DS that the pool belongs to.
    393  *
    394  *    attrs:
    395  *
    396  *    List of name value pairs corresponding to the attributes
    397  *    of the ZFS data store (e.g. zpool and dataset attribues).
    398  *    *Note: We may want to rethink how we are handling attributes.
    399  *	     We should probably have some general attributes (e.g.
    400  *	     online/offline, size, free_size).  Then we can have
    401  *	     attributes that are specific to the storage type (e.g.
    402  *	     zpool configuration (e.g. mirrored, RAIDZ),
    403  *	     dataset attributes (e.g. encryption, compression).
    404  */
    405 struct ds_zfsinfo {
    406       	ds_guid_map	guid_map;
    407 	ds_zfsattr	attrs<>;
    408 };
    409 
    410 /*
    411  * ds_storinfo -
    412  *
    413  *	Information about the storage available to the data server.
    414  *	Currently, the only storage type is ZFS, but there may be others
    415  *	in the future.
    416  */
    417 union ds_storinfo switch (storage_type type) {
    418 case ZFS:
    419 	ds_zfsinfo zfs_info;
    420 default:
    421 	void;
    422 };
    423 
    424 enum ds_attr_version {
    425 	DS_ATTR_v1 = 1
    426 };
    427 
    428 /*
    429  * DS_REPORTAVAILargs -
    430  *
    431  * A message to the MDS from a DS to provide availability
    432  * information for storage pools and network interfaces.
    433  *
    434  *    ds_id:
    435  *
    436  *    The short-hand idenifier assigned by the MDS and returned
    437  *    in the DS_EXIBI reply.
    438  *
    439  *    ds_addrs:
    440  *
    441  *    An array of DS_addr associated with DS and information
    442  *    about the intended use of each address; Specified as
    443  *    an array since it is possible for a DS to have multiple
    444  *    interfaces available.
    445  *
    446  *    ds_attrvers:
    447  *
    448  *    Version indicating the set of attributes that the data server is
    449  *    aware of/supports. (XXX - This may change to a supported attrs bitmask).
    450  *
    451  *    ds_storinfo:
    452  *
    453  *    Array of storage information.  For each piece of storage there will
    454  *    be one entry in this array.
    455  */
    456 struct DS_REPORTAVAILargs {
    457       	ds_id            ds_id;
    458       	ds_verifier	 ds_verifier;
    459       	struct ds_addr   ds_addrs<>;
    460 	ds_attr_version  ds_attrvers;
    461       	ds_storinfo      ds_storinfo<>;
    462 };
    463 
    464 /*
    465  * DS_REPORTAVAILres_ok -
    466  *
    467  * Response from the MDS on a successful DS_REPORTAVAIL call.
    468  *
    469  * ds_attrvers:
    470  *
    471  * Version indicating the set of attributes that the metadata server is
    472  * aware of/supports.  (XXX - This may change to a supported attrs bitmask).
    473  *
    474  * guid_map:
    475  *
    476  * Map of data server guids to Metadata Server Storage IDs (MDS SIDs)
    477  */
    478 struct DS_REPORTAVAILresok {
    479 	ds_attr_version	ds_attrvers;
    480       	ds_guid_map 	guid_map<>;
    481 };
    482 
    483 union DS_REPORTAVAILres switch (ds_status status) {
    484 case DS_OK:
    485       	DS_REPORTAVAILresok res_ok;
    486 default:
    487       	void;
    488 };
    489 
    490 /*
    491  * DS_SECINFO -
    492  *
    493  * A message from the DS to MDS used to inquire
    494  * for the secrity flavors of an object.
    495  *
    496  */
    497 struct DS_SECINFOargs {
    498       	nfs_fh4	object;
    499       	netaddr4	cl_addr;
    500 };
    501 
    502 /* RPCSEC_GSS has a value of '6' - See RFC 2203 */
    503 union ds_secinfo switch (uint32_t flavor) {
    504  case RPCSEC_GSS:
    505 	 rpcsec_gss_info	flavor_info;
    506  default:
    507 	 void;
    508 };
    509 
    510 typedef ds_secinfo DS_SECINFOresok<>;
    511 
    512 union DS_SECINFOres switch (ds_status status) {
    513 case DS_OK:
    514 	DS_SECINFOresok res_ok;
    515 default:
    516 	void;
    517 };
    518 
    519 /*
    520  * DS_SHUTDOWN -
    521  *
    522  * A notification to the MDS that this Data Server has/is in
    523  * the process of a graceful shutdown.
    524  *
    525  */
    526 struct DS_SHUTDOWNargs {
    527       	ds_id        ds_id;
    528 };
    529 
    530 
    531 struct DS_SHUTDOWNres {
    532       	ds_status status;
    533 };
    534 
    535 program PNFSCTLDS {
    536 	version PNFSCTLDS_V1 {
    537 		void
    538 		    DSPROC_NULL(void) = 0;
    539 
    540 		DS_CHECKSTATEres
    541 		    DS_CHECKSTATE(DS_CHECKSTATEargs) = 1;
    542 
    543 		DS_EXIBIres
    544 		    DS_EXIBI(DS_EXIBIargs) = 2;
    545 
    546 		DS_FMATPTres
    547 		    DS_FMATPT(DS_FMATPTargs) = 3;
    548 
    549 		DS_MAP_MDS_DATASET_IDres
    550 		    DS_MAP_MDS_DATASET_ID(DS_MAP_MDS_DATASET_IDargs) = 4;
    551 
    552 		DS_MAP_MDSSIDres
    553 		    DS_MAP_MDSSID(DS_MAP_MDSSIDargs) = 5;
    554 
    555 		DS_RENEWres
    556 		    DS_RENEW(DS_RENEWargs) = 6;
    557 
    558 		DS_REPORTAVAILres
    559 		    DS_REPORTAVAIL(DS_REPORTAVAILargs) = 7;
    560 
    561 		DS_SECINFOres
    562 		    DS_SECINFO(DS_SECINFOargs) = 8;
    563 
    564 		DS_SHUTDOWNres
    565 		    DS_SHUTDOWN(DS_SHUTDOWNargs) = 9;
    566 	} = 1;
    567 } = 104001;
    568 
    569 
    570 /*
    571  * NFS MDS Control Protocol:
    572  *
    573  * Traffic flows from the MDS to the data-server
    574  */
    575 
    576 struct ds_filesegbuf {
    577       	offset4	offset;
    578       	opaque	data<>;
    579 };
    580 
    581 struct ds_fileseg {
    582       	offset4 offset;
    583       	count4	count;
    584 };
    585 
    586 
    587 /*
    588  * DS_COMMIT:
    589  *
    590  * Commit a range written to a data-server.
    591  *
    592  */
    593 struct DS_COMMITargs {
    594       	nfs_fh4		fh;
    595       	ds_fileseg	cmv<>;
    596 };
    597 
    598 struct DS_COMMITresok {
    599       	ds_verifier	writeverf;
    600       	count4		count<>;
    601 };
    602 
    603 union DS_COMMITres switch (ds_status status) {
    604 case DS_OK:
    605       	DS_COMMITresok res_ok;
    606 default:
    607       	void;
    608 };
    609 
    610 /*
    611  * DS_GETATTR:
    612  *
    613  * Query data-server for attributes for the specified object.
    614  *
    615  *    fh:
    616  *
    617  *    The file handle for the object for which the DS
    618  *    is to give attributes for.
    619  *
    620  *    dattr:
    621  *
    622  *    Bitmap of attributes.
    623  */
    624 struct DS_GETATTRargs {
    625       	nfs_fh4 fh;
    626       	ds_attr dattrs;
    627 };
    628 
    629 union DS_GETATTRres switch (ds_status status) {
    630 case DS_OK:
    631       	ds_attr dattrs;
    632 default:
    633       	void;
    634 };
    635 
    636 /*
    637  * DS_INVALIDATE:
    638  *
    639  * Invalidate state at the data-server.
    640  *
    641  * The scope of invalidation is dependent on object type.
    642  *
    643  *   type:
    644  *
    645  *   represents the type (and by virtue scope) of state invalidation
    646  *   that should occur at the DS.
    647  *
    648  *
    649  */
    650 enum ds_invalidate_type {
    651       	DS_INVALIDATE_ALL,
    652       	DS_INVALIDATE_LAYOUT,
    653       	DS_INVALIDATE_MDS_DATASET_ID,
    654       	DS_INVALIDATE_CLIENTID,
    655       	DS_INVALIDATE_STATEID
    656 };
    657 
    658 struct ds_inval_stateid {
    659       	stateid4 stateid;
    660       	nfs_fh4 fh;
    661 };
    662 
    663 struct ds_inval_layout {
    664       	clientid4 mds_clid;
    665       	nfs_fh4 fh;
    666 };
    667 
    668 union DS_INVALIDATEargs switch (ds_invalidate_type obj) {
    669 
    670 case DS_INVALIDATE_LAYOUT:
    671       	ds_inval_layout layout;
    672 
    673 case DS_INVALIDATE_MDS_DATASET_ID:
    674       	mds_dataset_id	dataset_id;
    675 
    676 case DS_INVALIDATE_STATEID:
    677       	ds_inval_stateid stateid;
    678 
    679 case DS_INVALIDATE_CLIENTID:
    680       	clientid4  clid;
    681 
    682 case DS_INVALIDATE_ALL:
    683       	void;
    684 };
    685 
    686 struct DS_INVALIDATEres {
    687       	ds_status status;
    688 };
    689 
    690 /*
    691  * DS_LIST:
    692  *
    693  * Get a list of objects from the data-server matching provided
    694  * criteria. (either mds_dataset_id or mds_sid)
    695  */
    696 enum ds_list_type {
    697       	DS_LIST_MDS_SID,
    698       	DS_LIST_MDS_DATASET_ID
    699 };
    700 
    701 struct ds_list_sid_arg {
    702       	mds_sid mds_sid;
    703       	uint64_t cookie;
    704       	count4   maxcount;
    705 };
    706 
    707 struct ds_list_dataset_arg {
    708       	mds_dataset_id dataset_id;
    709       	uint64_t cookie;
    710       	count4 maxcount;
    711 };
    712 
    713 union DS_LISTargs switch (ds_list_type dla_type) {
    714 case DS_LIST_MDS_SID:
    715 	ds_list_sid_arg sid;
    716 
    717 case DS_LIST_MDS_DATASET_ID:
    718 	ds_list_dataset_arg dataset_id;
    719 
    720 default:
    721       	void;
    722 };
    723 
    724 struct DS_LISTresok {
    725       	nfs_fh4		fh_list<>;
    726       	uint64_t	cookie;
    727 };
    728 
    729 union DS_LISTres switch (ds_status status) {
    730 case DS_OK:
    731 	DS_LISTresok	res_ok;
    732 default:
    733 	void;
    734 };
    735 
    736 /*
    737  * MDS to DS - Data movement initiation messages
    738  */
    739 struct DS_OBJ_MOVEargs {
    740       	uint64_t taskid;
    741       	nfs_fh4 source;
    742       	nfs_fh4 target;
    743       	netaddr4 targetserver;
    744 };
    745 
    746 struct DS_OBJ_MOVEres {
    747       	ds_status status;
    748 };
    749 
    750 struct DS_OBJ_MOVE_STATUSargs {
    751       	uint64_t taskid;
    752 };
    753 
    754 struct DS_OBJ_MOVE_STATUSresok {
    755       	uint64_t maxoffset;
    756       	bool complete;
    757 };
    758 
    759 union DS_OBJ_MOVE_STATUSres switch (ds_status status) {
    760 case DS_OK:
    761 	DS_OBJ_MOVE_STATUSresok	res_ok;
    762 default:
    763 	void;
    764 };
    765 
    766 struct DS_OBJ_MOVE_ABORTargs {
    767       	uint64_t taskid;
    768 };
    769 
    770 struct DS_OBJ_MOVE_ABORTres {
    771       	ds_status status;
    772 };
    773 
    774 /*
    775  * DS_PNFSSTAT
    776  *
    777  * Return the kstat counters.
    778  *
    779  */
    780 
    781 /* RPC kstats */
    782 const DS_NFSSTAT_RPC     = 0x000000001;
    783 
    784 /* NFS kstats */
    785 const DS_NFSSTAT_NFS     = 0x000000002;
    786 
    787 /* the DMOV protocol kstats */
    788 const DS_NFSSTAT_DMOV    = 0x000000004;
    789 
    790 /* the control protocol kstats */
    791 const DS_NFSSTAT_CP      = 0x000000008;
    792 
    793 /* CPU kstat (all stats for module cpu)*/
    794 const DS_NFSSTAT_CPU     = 0x000000010;
    795 
    796 /*
    797  * More anticipated ...
    798  */
    799 struct DS_PNFSSTATargs {
    800       	uint64_t stat_wanted;
    801 };
    802 
    803 struct DS_PNFSSTATresok {
    804       	opaque      nvlist<>;
    805 };
    806 
    807 union DS_PNFSSTATres switch (ds_status status) {
    808 case DS_OK:
    809 	DS_PNFSSTATresok	res_ok;
    810 default:
    811 	void;
    812 };
    813 
    814 /*
    815  * DS_READ:
    816  *
    817  * Read a range of bytes from a data-server
    818  */
    819 struct DS_READargs {
    820       	nfs_fh4		fh;
    821       	count4		count;
    822       	ds_fileseg	rdv<>;
    823 };
    824 
    825 struct DS_READresok {
    826       	bool	eof;
    827       	count4	count;
    828       	ds_filesegbuf rdv<>;
    829 };
    830 
    831 union DS_READres switch (ds_status status) {
    832 case DS_OK:
    833       	DS_READresok res_ok;
    834 default:
    835       	void;
    836 };
    837 
    838 /*
    839  * DS_REMOVE:
    840  *
    841  * Remove object(s) or entire fsid at the data-server
    842  *
    843  */
    844 enum ds_rm_type {
    845       	DS_RM_OBJ,
    846       	DS_RM_MDS_DATASET_ID
    847 };
    848 
    849 union DS_REMOVEargs switch (ds_rm_type type) {
    850 case DS_RM_OBJ:
    851       	nfs_fh4		obj<>;
    852 case DS_RM_MDS_DATASET_ID:
    853       	mds_dataset_id  dataset_id<>;
    854 default:
    855       	void;
    856 };
    857 
    858 struct DS_REMOVEres {
    859       	ds_status	status;
    860 };
    861 
    862 /*
    863  * DS_SETATTR:
    864  *
    865  * Set/Store attributes for the specified object
    866  * at the data-server
    867  *
    868  *    fh:
    869  *
    870  *    The file handle for the object for which the DS
    871  *    is to set attributes for.
    872  *
    873  *    dattrs:
    874  *
    875  *    Bitmap of attributes.
    876  */
    877 struct DS_SETATTRargs {
    878       	nfs_fh4 fh;
    879       	ds_attr dattrs;
    880 };
    881 
    882 struct DS_SETATTRres {
    883       	ds_status status;
    884 };
    885 
    886 /*
    887  * DS_SNAP
    888  *
    889  * For a given MDS Dataset ID at the MDS, snapshot the data-set.
    890  *
    891  */
    892 struct DS_SNAPargs {
    893       	mds_dataset_id dataset_id;
    894 };
    895 
    896 struct DS_SNAPresok {
    897       	utf8string  name;
    898 };
    899 
    900 union DS_SNAPres switch (ds_status status) {
    901 case DS_OK:
    902       	DS_SNAPresok res_ok;
    903 default:
    904       	void;
    905 };
    906 
    907 /*
    908  * DS_STAT:
    909  *
    910  * Collect statistics for the status of an object,
    911  * like size etc..
    912  *
    913  */
    914 struct DS_STATargs {
    915       	nfs_fh4 object;
    916 };
    917 
    918 
    919 union DS_STATres switch (ds_status status) {
    920 case DS_OK:
    921       	ds_attr  dattr;
    922 default:
    923       	void;
    924 };
    925 
    926 /*
    927  * DS_WRITE:
    928  *
    929  * Write a range of bytes to a data-server
    930  *
    931  */
    932 struct DS_WRITEargs {
    933       	nfs_fh4		fh;
    934       	stable_how4	stable;
    935       	count4		count;
    936       	ds_filesegbuf	wrv<>;
    937 };
    938 
    939 struct DS_WRITEresok {
    940       	stable_how4	committed;
    941       	ds_verifier	writeverf;
    942       	count4		wrv<>;
    943 };
    944 
    945 union DS_WRITEres switch (ds_status status) {
    946 case DS_OK:
    947       	DS_WRITEresok res_ok;
    948 default:
    949       	void;
    950 };
    951 
    952 program PNFSCTLMDS {
    953 	version PNFSCTLMDS_V1 {
    954 
    955 		void
    956 		    DSPROC_NULL(void) = 0;
    957 
    958 		DS_COMMITres
    959 		    DS_COMMIT(DS_COMMITargs) = 1;
    960 
    961 		DS_GETATTRres
    962 		    DS_GETATTR(DS_GETATTRargs) = 2;
    963 
    964 		DS_INVALIDATEres
    965 		    DS_INVALIDATE(DS_INVALIDATEargs) = 3;
    966 
    967 		DS_LISTres
    968 		    DS_LIST(DS_LISTargs) = 4;
    969 
    970 		DS_OBJ_MOVEres
    971 		    DS_OBJ_MOVE(DS_OBJ_MOVEargs) = 5;
    972 
    973 		DS_OBJ_MOVE_ABORTres
    974 		    DS_OBJ_MOVE_ABORT(DS_OBJ_MOVE_ABORTargs) = 6;
    975 
    976 		DS_OBJ_MOVE_STATUSres
    977 		    DS_OBJ_MOVE_STATUS(DS_OBJ_MOVE_STATUSargs) = 7;
    978 
    979 		DS_PNFSSTATres
    980 		    DS_PNFSSTAT(DS_PNFSSTATargs) = 8;
    981 
    982 		DS_READres
    983 		    DS_READ(DS_READargs) = 9;
    984 
    985 		DS_REMOVEres
    986 		    DS_REMOVE(DS_REMOVEargs) = 10;
    987 
    988 		DS_SETATTRres
    989 		    DS_SETATTR(DS_SETATTRargs) = 11;
    990 
    991 		DS_STATres
    992 		    DS_STAT(DS_STATargs) = 12;
    993 
    994 		DS_SNAPres
    995 		    DS_SNAP(DS_SNAPargs) = 13;
    996 
    997 		DS_WRITEres
    998 		    DS_WRITE(DS_WRITEargs) = 14;
    999 
   1000 	} = 1;
   1001 } = 104000;
   1002 
   1003 /*
   1004  * DS to DS data movement protocol.
   1005  */
   1006 
   1007 struct MOVEargs {
   1008 	uint64_t taskid;
   1009 	uint64_t minoffset;
   1010 	uint64_t maxoffset;
   1011 	uint32_t maxbytes;
   1012 };
   1013 
   1014 struct MOVEsegment {
   1015 	uint64_t fileoffset;
   1016 	uint32_t len;
   1017 };
   1018 
   1019 struct MOVEresok {
   1020 	opaque data<>;
   1021 	struct MOVEsegment segments<>;
   1022 };
   1023 
   1024 union MOVEres switch (ds_status status) {
   1025 case DS_OK:
   1026 	MOVEresok res_ok;
   1027 default:
   1028 	void;
   1029 };
   1030 
   1031 program PNFSCTLMV {
   1032 	version PNFSCTLMV_V1 {
   1033 		void
   1034 		    DSPROC_NULL(void) = 0;
   1035 
   1036 		MOVEres
   1037 		    MOVE(MOVEargs) = 1;
   1038 	} = 1;
   1039 } = 104002;
   1040