Tryag File Manager

//usr/lib/vmware-tools/modules/source/vmxnet3.tar

vmxnet3-only/���������������������������������������������������������������������������������������0000755�0000000�0000000�00000000000�13432726376�012154� 5����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/Makefile�������������������������������������������������������������������������������0000644�0000000�0000000�00000007406�13432725347�013620� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/usr/bin/make -f
##########################################################
# Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation version 2 and no later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
#
##########################################################

####
####  VMware kernel module Makefile to be distributed externally
####

####
#### SRCROOT _must_ be a relative path.
####
SRCROOT = .

#
# open-vm-tools doesn't replicate shared source files for different modules;
# instead, files are kept in shared locations. So define a few useful macros
# to be able to handle both cases cleanly.
#
INCLUDE      :=
ifdef OVT_SOURCE_DIR
AUTOCONF_DIR := $(OVT_SOURCE_DIR)/modules/linux/shared/autoconf
VMLIB_PATH   = $(OVT_SOURCE_DIR)/lib/$(1)
INCLUDE      += -I$(OVT_SOURCE_DIR)/modules/linux/shared
INCLUDE      += -I$(OVT_SOURCE_DIR)/lib/include
else
AUTOCONF_DIR := $(SRCROOT)/shared/autoconf
INCLUDE      += -I$(SRCROOT)/shared
endif

VM_UNAME = $(shell uname -r)

# Header directory for the running kernel
ifdef LINUXINCLUDE
HEADER_DIR = $(LINUXINCLUDE)
else
HEADER_DIR = /lib/modules/$(VM_UNAME)/build/include
endif

BUILD_DIR = $(HEADER_DIR)/..

DRIVER := vmxnet3
PRODUCT := drivers-for-linux-guest

# Grep program
GREP = /bin/grep

vm_check_gcc = $(shell if $(CC) $(1) -S -o /dev/null -xc /dev/null \
        > /dev/null 2>&1; then echo "$(1)"; else echo "$(2)"; fi)
vm_check_file = $(shell if test -f $(1); then echo "yes"; else echo "no"; fi)

ifndef VM_KBUILD
VM_KBUILD := no
ifeq ($(call vm_check_file,$(BUILD_DIR)/Makefile), yes)
VM_KBUILD := yes
endif
export VM_KBUILD
endif

ifndef VM_KBUILD_SHOWN
ifeq ($(VM_KBUILD), no)
VM_DUMMY := $(shell echo >&2 "Using standalone build system.")
else
VM_DUMMY := $(shell echo >&2 "Using kernel build system.")
endif
VM_KBUILD_SHOWN := yes
export VM_KBUILD_SHOWN
endif

ifneq ($(VM_KBUILD), no)

# If there is no version defined, we are in toplevel pass, not yet in kernel makefiles...
ifeq ($(VERSION),)

DRIVER_KO := $(DRIVER).ko

.PHONY: $(DRIVER_KO)

auto-build: $(DRIVER_KO)
	cp -f $< $(SRCROOT)/../$(DRIVER).o

# $(DRIVER_KO) is a phony target, so compare file times explicitly
$(DRIVER): $(DRIVER_KO)
	if [ $< -nt $@ ] || [ ! -e $@ ] ; then cp -f $< $@; fi

#
# Define a setup target that gets built before the actual driver.
# This target may not be used at all, but if it is then it will be defined
# in Makefile.kernel
#
prebuild:: ;
postbuild:: ;

$(DRIVER_KO): prebuild
	$(MAKE) -C $(BUILD_DIR) SUBDIRS=$$PWD SRCROOT=$$PWD/$(SRCROOT) \
	  MODULEBUILDDIR=$(MODULEBUILDDIR) modules
	$(MAKE) -C $$PWD SRCROOT=$$PWD/$(SRCROOT) \
	  MODULEBUILDDIR=$(MODULEBUILDDIR) postbuild
endif

vm_check_build = $(shell if $(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) \
	$(CPPFLAGS) $(CFLAGS) $(CFLAGS_KERNEL) $(LINUXINCLUDE) \
	$(EXTRA_CFLAGS) -Iinclude2/asm/mach-default \
	-DKBUILD_BASENAME=\"$(DRIVER)\" \
	-Werror -S -o /dev/null -xc $(1) \
	> /dev/null 2>&1; then echo "$(2)"; else echo "$(3)"; fi)

CC_WARNINGS := -Wall -Wstrict-prototypes
CC_OPTS := $(GLOBAL_DEFS) $(CC_WARNINGS) -DVMW_USING_KBUILD
ifdef VMX86_DEVEL
CC_OPTS += -DVMX86_DEVEL
endif
ifdef VMX86_DEBUG
CC_OPTS += -DVMX86_DEBUG
endif

include $(SRCROOT)/Makefile.kernel

else

include $(SRCROOT)/Makefile.normal

endif

#.SILENT:
����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/vmxnet3_int.h��������������������������������������������������������������������������0000444�0000000�0000000�00000032241�13432725331�014571� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef _VMXNET3_INT_H
#define _VMXNET3_INT_H

#define INCLUDE_ALLOW_MODULE
#include "includeCheck.h"

#include "compat_slab.h"
#include "compat_spinlock.h"
#include "compat_ioport.h"
#include "compat_pci.h"
#include "compat_highmem.h"
#include "compat_timer.h"
#include "compat_netdevice.h"
#include "compat_skbuff.h"
#include "compat_interrupt.h"
#include "compat_workqueue.h"
#include "compat_module.h"
#include "compat_log2.h"
#include "compat_ethtool.h"

#include <asm/dma.h>
#include <asm/page.h>
#include <asm/uaccess.h>
#include <linux/types.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/in.h>
#include <linux/init.h>
#include <linux/etherdevice.h>
#include <linux/bitops.h>
#include <linux/ethtool.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <asm/checksum.h>
#include <linux/moduleparam.h>

#include <linux/if_vlan.h>
#include <linux/if_arp.h>
#include <linux/inetdevice.h>

#include <net/ip.h>
#include <net/udp.h>

#ifdef CONFIG_COMPAT
#ifndef HAVE_UNLOCKED_IOCTL
#include <linux/ioctl32.h>
#endif
#endif

#include "vm_basic_types.h"
#include "vmxnet3_defs.h"
#include "vmxnet3_version.h"
#include <linux/stddef.h>

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 0) && !defined(VMXNET3_NO_NAPI)
#   define VMXNET3_NAPI
#endif

#ifdef VLAN_GROUP_ARRAY_SPLIT_PARTS
#define compat_vlan_group_get_device(vlan_grp, vid)	vlan_group_get_device(vlan_grp, vid)
#define compat_vlan_group_set_device(vlan_grp, vid, dev)	vlan_group_set_device(vlan_grp, vid, dev)
#else
#define compat_vlan_group_get_device(vlan_grp, vid)	((vlan_grp)->vlan_devices[(vid)])
#define compat_vlan_group_set_device(vlan_grp, vid, dev)	((vlan_grp)->vlan_devices[(vid)] = (dev))
#endif

#ifdef VMXNET3_NAPI
#   ifdef VMX86_DEBUG
#      define VMXNET3_DRIVER_VERSION_REPORT VMXNET3_DRIVER_VERSION_STRING"-NAPI(debug)"
#   else
#      define VMXNET3_DRIVER_VERSION_REPORT VMXNET3_DRIVER_VERSION_STRING"-NAPI"
#   endif
#else
#   ifdef VMX86_DEBUG
#      define VMXNET3_DRIVER_VERSION_REPORT VMXNET3_DRIVER_VERSION_STRING"(debug)"
#   else
#      define VMXNET3_DRIVER_VERSION_REPORT VMXNET3_DRIVER_VERSION_STRING
#   endif
#endif

#if defined(CONFIG_PCI_MSI) && LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
	/* RSS only makes sense if MSI-X and new NAPI is supported. */
	#define VMXNET3_RSS
#endif

struct vmxnet3_cmd_ring {
	Vmxnet3_GenericDesc *base;
	u32		size;
	u32		next2fill;
	u32		next2comp;
	u8		gen;
	dma_addr_t           basePA;
};

static inline void
vmxnet3_cmd_ring_adv_next2fill(struct vmxnet3_cmd_ring *ring)
{
	ring->next2fill++;
	if (unlikely(ring->next2fill == ring->size)) {
		ring->next2fill = 0;
		VMXNET3_FLIP_RING_GEN(ring->gen);
	}
}

static inline void
vmxnet3_cmd_ring_adv_next2comp(struct vmxnet3_cmd_ring *ring)
{
	VMXNET3_INC_RING_IDX_ONLY(ring->next2comp, ring->size);
}

static inline int
vmxnet3_cmd_ring_desc_avail(struct vmxnet3_cmd_ring *ring)
{
	return (ring->next2comp > ring->next2fill ? 0 : ring->size) +
		ring->next2comp - ring->next2fill - 1;
}

struct vmxnet3_comp_ring {
	Vmxnet3_GenericDesc *base;
	u32               size;
	u32               next2proc;
	u8                gen;
	u8                intr_idx;
	dma_addr_t        basePA;
};

static inline void
vmxnet3_comp_ring_adv_next2proc(struct vmxnet3_comp_ring *ring)
{
	ring->next2proc++;
	if (unlikely(ring->next2proc == ring->size)) {
		ring->next2proc = 0;
		VMXNET3_FLIP_RING_GEN(ring->gen);
	}
}

struct vmxnet3_tx_data_ring {
	Vmxnet3_TxDataDesc *base;
	u32                 size;
	dma_addr_t          basePA;
};

enum vmxnet3_buf_map_type {
	VMXNET3_MAP_INVALID = 0,
	VMXNET3_MAP_NONE,
	VMXNET3_MAP_SINGLE,
	VMXNET3_MAP_PAGE,
};

struct vmxnet3_tx_buf_info {
	u32      map_type;
	u16      len;
	u16      sop_idx;
	dma_addr_t  dma_addr;
	struct sk_buff *skb;
};

struct vmxnet3_tq_driver_stats {
	u64 drop_total;     /* # of pkts dropped by the driver, the
                           * counters below track droppings due to
                           * different reasons
                           */
	u64 drop_too_many_frags;
	u64 drop_oversized_hdr;
	u64 drop_hdr_inspect_err;
	u64 drop_tso;

u64 tx_ring_full;
	u64 linearized;         /* # of pkts linearized */
	u64 copy_skb_header;    /* # of times we have to copy skb header */
	u64 oversized_hdr;
};

struct vmxnet3_tx_ctx {
	Bool   ipv4;
	Bool   ipv6;
	u16 mss;
	u32 eth_ip_hdr_size; /* only valid for pkts requesting tso or csum
				 * offloading
				 */
	u32 l4_hdr_size;     /* only valid if mss != 0 */
	u32 copy_size;       /* # of bytes copied into the data ring */
	Vmxnet3_GenericDesc *sop_txd;
	Vmxnet3_GenericDesc *eop_txd;
};

struct vmxnet3_tx_queue {
	struct vmxnet3_adapter		*adapter;
	spinlock_t                      tx_lock;
	struct vmxnet3_cmd_ring         tx_ring;
	struct vmxnet3_tx_buf_info     *buf_info;
	dma_addr_t                      buf_info_pa;
	struct vmxnet3_tx_data_ring     data_ring;
	struct vmxnet3_comp_ring        comp_ring;
	Vmxnet3_TxQueueCtrl            *shared;
	struct vmxnet3_tq_driver_stats  stats;
	Bool                            stopped;
	int                             num_stop;  /* # of times the queue is
						    * stopped */
	int				qid;
	char				name[IFNAMSIZ+8]; /* To identify interrupt*/
} __attribute__((__aligned__(SMP_CACHE_BYTES)));

enum vmxnet3_rx_buf_type {
	VMXNET3_RX_BUF_NONE = 0,
	VMXNET3_RX_BUF_SKB = 1,
	VMXNET3_RX_BUF_PAGE = 2
};

struct vmxnet3_rx_buf_info {
	enum vmxnet3_rx_buf_type buf_type;
	u16     len;
	union {
		struct sk_buff *skb;
		struct page    *page;
		unsigned long   shm_idx;
	};
	dma_addr_t dma_addr;
};

struct vmxnet3_rx_ctx {
	struct sk_buff *skb;
	u32 sop_idx;
};

struct vmxnet3_rq_driver_stats {
	u64 drop_total;
	u64 drop_err;
	u64 drop_fcs;
	u64 rx_buf_alloc_failure;
};

struct vmxnet3_rx_queue {
	struct vmxnet3_adapter	  *adapter;
#ifdef VMXNET3_NAPI
	struct napi_struct        napi;
#endif
	struct vmxnet3_cmd_ring   rx_ring[2];
	struct vmxnet3_comp_ring  comp_ring;
	struct vmxnet3_rx_ctx     rx_ctx;
	u32 qid;            /* rqID in RCD for buffer from 1st ring */
	u32 qid2;           /* rqID in RCD for buffer from 2nd ring */
	u32 uncommitted[2]; /* # of buffers allocated since last RXPROD
			     * update */
	struct vmxnet3_rx_buf_info     *buf_info[2];
	dma_addr_t                      buf_info_pa;
	Vmxnet3_RxQueueCtrl            *shared;
	struct vmxnet3_rq_driver_stats  stats;
	char			  name[IFNAMSIZ+8]; /* To identify interrupt*/
} __attribute__((__aligned__(SMP_CACHE_BYTES)));

#define VMXNET3_DEVICE_MAX_TX_QUEUES 8
#define VMXNET3_DEVICE_MAX_RX_QUEUES 8   /* Keep this value as a power of 2 */

/* Should be less than UPT1_RSS_MAX_IND_TABLE_SIZE */
#define VMXNET3_RSS_IND_TABLE_SIZE (VMXNET3_DEVICE_MAX_RX_QUEUES * 4)

#define VMXNET3_LINUX_MAX_MSIX_VECT     (VMXNET3_DEVICE_MAX_TX_QUEUES + \
					 VMXNET3_DEVICE_MAX_RX_QUEUES + 1)
#define VMXNET3_LINUX_MIN_MSIX_VECT     2 /* 1 for tx-rx pair and 1 for event */

struct vmxnet3_intr {
	enum vmxnet3_intr_mask_mode  mask_mode;
	enum vmxnet3_intr_type       type;          /* MSI-X, MSI, or INTx? */
	u8  num_intrs;			/* # of intr vectors */
	u8  event_intr_idx;		/* idx of the intr vector for event */
	u8  mod_levels[VMXNET3_LINUX_MAX_MSIX_VECT]; /* moderation level */
	char	event_msi_vector_name[IFNAMSIZ+11];
#ifdef CONFIG_PCI_MSI
	struct msix_entry msix_entries[VMXNET3_LINUX_MAX_MSIX_VECT];
#endif
};

enum {
	vmxnet3_intr_noshare = 0,	/* each queue has its own intr vector */
	vmxnet3_intr_txshare,		/* All tx queues share one intr */
	vmxnet3_intr_buddyshare,	/* Corresponding tx and rx queues share */
};

#ifdef VMXNET3_RSS
struct vmxnet3_rssinfo {
	int indices;
	int mask;
};
#endif

#define VMXNET3_STATE_BIT_RESETTING   0
#define VMXNET3_STATE_BIT_QUIESCED    1
struct vmxnet3_adapter {
	struct vmxnet3_tx_queue		tx_queue[VMXNET3_DEVICE_MAX_TX_QUEUES];
	struct vmxnet3_rx_queue		rx_queue[VMXNET3_DEVICE_MAX_RX_QUEUES];
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37)
	struct vlan_group		*vlan_grp;
#else
	unsigned long			active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
#endif
	struct vmxnet3_intr		intr;
	spinlock_t			cmd_lock;
	Vmxnet3_DriverShared		*shared;
	struct UPT1_RSSConf		*rss_conf;
	Vmxnet3_PMConf			*pm_conf;
	Vmxnet3_TxQueueDesc		*tqd_start;     /* all tx queue desc */
	Vmxnet3_RxQueueDesc		*rqd_start;	/* all rx queue desc */
	struct net_device		*netdev;
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 22)
	struct net_device_stats		net_stats;
#endif
	struct pci_dev			*pdev;

u8				*hw_addr0; /* for BAR 0 */
	u8				*hw_addr1; /* for BAR 1 */
	u8                              version;
	/* feature control */
	Bool				rxcsum;
	Bool				lro;
#ifdef VMXNET3_RSS
	Bool				rss;
	struct vmxnet3_rssinfo		rssinfo;
#endif
	uint32				num_rx_queues;
	uint32				num_tx_queues;

/* rx buffer related */
	unsigned   skb_buf_size;
	int        rx_buf_per_pkt;  /* only apply to the 1st ring */
	dma_addr_t shared_pa;
	dma_addr_t queue_desc_pa;

/* Wake-on-LAN */
	u32     wol;

/* Link speed */
	u32     link_speed; /* in mbps */

u64     tx_timeout_count;
	compat_work reset_work;
	compat_work resize_ring_work;

unsigned long  state;    /* VMXNET3_STATE_BIT_xxx */

int dev_number;
	Bool is_shm;
	int share_intr;
	struct vmxnet3_shm_pool *shm;

struct timer_list drop_check_timer;
	u64 prev_oob_drop_count;
	u64 prev_udp_drop_count;
	u32 drop_counter;
	u32 no_drop_counter;
	u32 new_rx_ring_size;
	u32 drop_check_delay;
	Bool use_adaptive_ring;

dma_addr_t adapter_pa;
	dma_addr_t pm_conf_pa;
	dma_addr_t rss_conf_pa;
};

struct vmxnet3_stat_desc {
	char desc[ETH_GSTRING_LEN];
	int  offset;
};

#define VMXNET3_WRITE_BAR0_REG(adapter, reg, val)  \
	writel(cpu_to_le32(val), (adapter)->hw_addr0 + (reg))
#define VMXNET3_READ_BAR0_REG(adapter, reg)        \
	le32_to_cpu(readl((adapter)->hw_addr0 + (reg)))

#define VMXNET3_WRITE_BAR1_REG(adapter, reg, val)  \
	writel(cpu_to_le32(val), (adapter)->hw_addr1 + (reg))
#define VMXNET3_READ_BAR1_REG(adapter, reg)        \
	le32_to_cpu(readl((adapter)->hw_addr1 + (reg)))

#define VMXNET3_WAKE_QUEUE_THRESHOLD(tq)  (5)
#define VMXNET3_RX_ALLOC_THRESHOLD(rq, ring_idx, adapter) \
	((rq)->rx_ring[ring_idx].size >> 3)

#define VMXNET3_GET_ADDR_LO(dma)   ((u32)(dma))
#define VMXNET3_GET_ADDR_HI(dma)   ((u32)(((u64)(dma)) >> 32))

/* must be a multiple of VMXNET3_RING_SIZE_ALIGN */
#define VMXNET3_DEF_TX_RING_SIZE    512
#define VMXNET3_DEF_RX_RING_SIZE    256

/* FIXME: what's the right value for this? */
#define VMXNET3_MAX_ETH_HDR_SIZE    22

#define VMXNET3_MAX_SKB_BUF_SIZE    (3*1024)

static inline void
set_flag_le16(__le16 *data, u16 flag)
{
	*data = cpu_to_le16(le16_to_cpu(*data) | flag);
}

static inline void
set_flag_le64(__le64 *data, u64 flag)
{
	*data = cpu_to_le64(le64_to_cpu(*data) | flag);
}

static inline void
reset_flag_le64(__le64 *data, u64 flag)
{
	*data = cpu_to_le64(le64_to_cpu(*data) & ~flag);
}

static inline Bool
vmxnet3_cmd_ring_desc_empty(struct vmxnet3_rx_queue *rq, u32 ring_idx)
{
   struct vmxnet3_cmd_ring *ring = rq->rx_ring + ring_idx;
   u32 next2comp = ring->next2comp;

while (rq->buf_info[ring_idx][next2comp].buf_type != VMXNET3_RX_BUF_SKB) {
      VMXNET3_INC_RING_IDX_ONLY(next2comp, ring->size);
   }

return (next2comp == ring->next2fill);
}

int
vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, struct
		vmxnet3_adapter *adapter, struct net_device *netdev);
int
vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter);

int
vmxnet3_activate_dev(struct vmxnet3_adapter *adapter);

void
vmxnet3_force_close(struct vmxnet3_adapter *adapter);

void
vmxnet3_reset_dev(struct vmxnet3_adapter *adapter);

void
vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter);

void
vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter);

int
vmxnet3_create_queues(struct vmxnet3_adapter *adapter,
		      u32 tx_ring_size, u32 rx_ring_size, u32 rx_ring2_size);

void
vmxnet3_vlan_features(struct vmxnet3_adapter *adapter, u16 vid, Bool allvids);

int
vmxnet3_set_ringsize(struct net_device *netdev, u32 new_tx_ring_size,
                     u32 new_rx_ring_size, u32 new_rx_ring2_size);

extern void vmxnet3_set_ethtool_ops(struct net_device *netdev);

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)
extern struct rtnl_link_stats64 *
vmxnet3_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats);
#else
extern struct net_device_stats *vmxnet3_get_stats(struct net_device *netdev);
#endif

extern char vmxnet3_driver_name[];

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
extern int vmxnet3_set_features(struct net_device *netdev, compat_netdev_features_t features);
#endif

/************************************************************
 *
 *   net.h
 *
 *   This file should contain all network global defines.
 *   No vlance/vmxnet/vnet/vmknet specific stuff should be
 *   put here only defines used/usable by all network code.
 *   --gustav
 *
 ************************************************************/

#ifndef VMWARE_DEVICES_NET_H
#define VMWARE_DEVICES_NET_H

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMCORE

#include "includeCheck.h"
#include "vm_device_version.h"

#ifdef VMCORE
#include "config.h"
#include "str.h"
#include "strutil.h"
#endif

#define ETHERNET_MTU         1518

#ifndef ETHER_ADDR_LEN
#define ETHER_ADDR_LEN          6  /* length of MAC address */
#endif
#define ETH_HEADER_LEN	       14  /* length of Ethernet header */
#define IP_ADDR_LEN	        4  /* length of IPv4 address */
#define IP_HEADER_LEN	       20  /* minimum length of IPv4 header */

#define ETHER_MAX_QUEUED_PACKET 1600

/*
 * State's that a NIC can be in currently we only use this
 * in VLance but if we implement/emulate new adapters that
 * we also want to be able to morph a new corresponding
 * state should be added.
 */

#define LANCE_CHIP  0x2934
#define VMXNET_CHIP 0x4392

/*
 * Size of reserved IO space needed by the LANCE adapter and
 * the VMXNET adapter. If you add more ports to Vmxnet than
 * there is reserved space you must bump VMXNET_CHIP_IO_RESV_SIZE.
 * The sizes must be powers of 2.
 */

#define LANCE_CHIP_IO_RESV_SIZE  0x20
#define VMXNET_CHIP_IO_RESV_SIZE 0x40

#define MORPH_PORT_SIZE 4

#ifdef VMCORE
typedef struct Net_AdapterCount {
   uint8 vlance;
   uint8 vmxnet2;
   uint8 vmxnet3;
   uint8 vrdma;
   uint8 e1000;
   uint8 e1000e;
} Net_AdapterCount;
#endif

#ifdef USERLEVEL

/*
 *----------------------------------------------------------------------------
 *
 * Net_AddAddrToLADRF --
 *
 *      Given a MAC address, sets the corresponding bit in the LANCE style
 *      Logical Address Filter 'ladrf'.
 *      The caller should have initialized the ladrf to all 0's, as this
 *      function only ORs on a bit in the array.
 *      'addr' is presumed to be ETHER_ADDR_LEN in size;
 *      'ladrf' is presumed to point to a 64-bit vector.
 *
 *      Derived from a long history of derivations, originally inspired by
 *      sample code from the AMD "Network Products: Ethernet Controllers 1998
 *      Data Book, Book 2", pages 1-53..1-55.
 *
 * Returns:
 *      None.
 *
 * Side effects:
 *      Updates 'ladrf'.
 *
 *----------------------------------------------------------------------------
 */

static INLINE void
Net_AddAddrToLadrf(const uint8 *addr,  // IN: pointer to MAC address
                   uint8 *ladrf)       // IN/OUT: pointer to ladrf
{
#define CRC_POLYNOMIAL_BE 0x04c11db7UL	/* Ethernet CRC, big endian */

uint16 hashcode;
   int32 crc = 0xffffffff;		/* init CRC for each address */
   int32 j;
   int32 bit;
   int32 byte;

ASSERT(addr);
   ASSERT(ladrf);

for (byte = 0; byte < ETHER_ADDR_LEN; byte++) {  /* for each address byte */
      /* process each address bit */
      for (bit = *addr++, j = 0;
           j < 8;
           j++, bit >>= 1) {
	 crc = (crc << 1) ^ ((((crc < 0 ? 1 : 0) ^ bit) & 0x01) ?
               CRC_POLYNOMIAL_BE : 0);
      }
   }
   hashcode = (crc & 1);	       /* hashcode is 6 LSb of CRC ... */
   for (j = 0; j < 5; j++) {	       /* ... in reverse order. */
      hashcode = (hashcode << 1) | ((crc>>=1) & 1);
   }

ladrf[hashcode >> 3] |= 1 << (hashcode & 0x07);
}
#endif // USERLEVEL

#ifdef VMCORE
/*
 *----------------------------------------------------------------------
 *
 * Net_GetNumAdapters --
 *
 *      Returns the number of each type of network adapter configured in this 
 *      VM.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE void
Net_GetNumAdapters(Net_AdapterCount *counts)
{
   uint32 i;

counts->vlance = 0;
   counts->vmxnet2 = 0;
   counts->vmxnet3 = 0;
   counts->vrdma = 0;
   counts->e1000 = 0;
   counts->e1000e = 0;

for (i = 0; i < MAX_ETHERNET_CARDS; i++) {
      char* adapterStr;

if (!Config_GetBool(FALSE, "ethernet%d.present", i)) {
	 continue;
      }
      adapterStr = Config_GetString("vlance", "ethernet%d.virtualDev", i);
      if (Str_Strcasecmp(adapterStr, "vmxnet3") == 0) {
         counts->vmxnet3++;
      } else if (Str_Strcasecmp(adapterStr, "vrdma") == 0) {
         counts->vrdma++;
      } else if (Str_Strcasecmp(adapterStr, "vlance") == 0) {
         counts->vlance++;
      } else if (Str_Strcasecmp(adapterStr, "vmxnet") == 0) {
         counts->vmxnet2++;
      } else if (Str_Strcasecmp(adapterStr, "e1000") == 0) {
         counts->e1000++;
      } else if (Str_Strcasecmp(adapterStr, "e1000e") == 0) {
         counts->e1000e++;
      } else {
         LOG_ONCE(("%s: unknown adapter: %s\n", __FUNCTION__, adapterStr));
      }
      free(adapterStr);
   }
}

#endif // VMCORE

#endif // VMWARE_DEVICES_NET_H
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/vmxnet3_ethtool.c����������������������������������������������������������������������0000444�0000000�0000000�00000065006�13432725331�015455� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "driver-config.h"
#include "compat_module.h"
#include <linux/moduleparam.h>
#include "vm_basic_types.h"
#include "vmxnet3_int.h"
#include <linux/pm.h>

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 39)
static u32
vmxnet3_get_rx_csum(struct net_device *netdev)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	return adapter->rxcsum;
}

static int
vmxnet3_set_rx_csum(struct net_device *netdev, u32 val)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	unsigned long flags;

if (adapter->rxcsum != val) {
		adapter->rxcsum = val;
		if (netif_running(netdev)) {
			if (val)
				set_flag_le64(
				&adapter->shared->devRead.misc.uptFeatures,
				UPT1_F_RXCSUM);
			else
				reset_flag_le64(
				&adapter->shared->devRead.misc.uptFeatures,
				UPT1_F_RXCSUM);

spin_lock_irqsave(&adapter->cmd_lock, flags);
			VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
					       VMXNET3_CMD_UPDATE_FEATURE);
			spin_unlock_irqrestore(&adapter->cmd_lock, flags);
		}
	}
	return 0;
}

int vmxnet3_set_tso(struct net_device *dev, u32 data) {
	if (data)
		dev->features |= COMPAT_NETIF_F_TSO;
	else
		dev->features &= ~COMPAT_NETIF_F_TSO;

return 0;
}

static u32
vmxnet3_get_tx_csum(struct net_device *netdev)
{
        return (netdev->features & NETIF_F_HW_CSUM) != 0;
}

static int
vmxnet3_set_tx_csum(struct net_device *netdev, u32 val)
{
        struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
        if (val) {
                netdev->features |= NETIF_F_HW_CSUM;
        } else {
                netdev->features &= ~ NETIF_F_HW_CSUM;
        }
        vmxnet3_vlan_features(adapter, 0, TRUE);
        return 0;
}

#endif

/* per tq stats maintained by the device */
static const struct vmxnet3_stat_desc
vmxnet3_tq_dev_stats[] = {
	/* description,         offset */
	{ "Tx Queue#",        0 },
	{ "  TSO pkts tx",        offsetof(UPT1_TxStats, TSOPktsTxOK) },
	{ "  TSO bytes tx",       offsetof(UPT1_TxStats, TSOBytesTxOK) },
	{ "  ucast pkts tx",      offsetof(UPT1_TxStats, ucastPktsTxOK) },
	{ "  ucast bytes tx",     offsetof(UPT1_TxStats, ucastBytesTxOK) },
	{ "  mcast pkts tx",      offsetof(UPT1_TxStats, mcastPktsTxOK) },
	{ "  mcast bytes tx",     offsetof(UPT1_TxStats, mcastBytesTxOK) },
	{ "  bcast pkts tx",      offsetof(UPT1_TxStats, bcastPktsTxOK) },
	{ "  bcast bytes tx",     offsetof(UPT1_TxStats, bcastBytesTxOK) },
	{ "  pkts tx err",        offsetof(UPT1_TxStats, pktsTxError) },
	{ "  pkts tx discard",    offsetof(UPT1_TxStats, pktsTxDiscard) },
};

/* per tq stats maintained by the driver */
static const struct vmxnet3_stat_desc
vmxnet3_tq_driver_stats[] = {
	/* description,         offset */
	{ "  drv dropped tx total", offsetof(struct vmxnet3_tq_driver_stats,
					drop_total) },
	{ "    too many frags",  offsetof(struct vmxnet3_tq_driver_stats,
					drop_too_many_frags) },
	{ "    giant hdr",       offsetof(struct vmxnet3_tq_driver_stats,
					drop_oversized_hdr) },
	{ "    hdr err",         offsetof(struct vmxnet3_tq_driver_stats,
					drop_hdr_inspect_err) },
	{ "    tso",             offsetof(struct vmxnet3_tq_driver_stats,
					drop_tso) },
	{ "  ring full",          offsetof(struct vmxnet3_tq_driver_stats,
					tx_ring_full) },
	{ "  pkts linearized",    offsetof(struct vmxnet3_tq_driver_stats,
					linearized) },
	{ "  hdr cloned",         offsetof(struct vmxnet3_tq_driver_stats,
					copy_skb_header) },
	{ "  giant hdr",          offsetof(struct vmxnet3_tq_driver_stats,
					oversized_hdr) },
};

/* per rq stats maintained by the device */
static const struct vmxnet3_stat_desc
vmxnet3_rq_dev_stats[] = {
	{ "Rx Queue#",        0 },
	{ "  LRO pkts rx",        offsetof(UPT1_RxStats, LROPktsRxOK) },
	{ "  LRO byte rx",        offsetof(UPT1_RxStats, LROBytesRxOK) },
	{ "  ucast pkts rx",      offsetof(UPT1_RxStats, ucastPktsRxOK) },
	{ "  ucast bytes rx",     offsetof(UPT1_RxStats, ucastBytesRxOK) },
	{ "  mcast pkts rx",      offsetof(UPT1_RxStats, mcastPktsRxOK) },
	{ "  mcast bytes rx",     offsetof(UPT1_RxStats, mcastBytesRxOK) },
	{ "  bcast pkts rx",      offsetof(UPT1_RxStats, bcastPktsRxOK) },
	{ "  bcast bytes rx",     offsetof(UPT1_RxStats, bcastBytesRxOK) },
	{ "  pkts rx out of buf", offsetof(UPT1_RxStats, pktsRxOutOfBuf) },
	{ "  pkts rx err",        offsetof(UPT1_RxStats, pktsRxError) },
};

/* per rq stats maintained by the driver */
static const struct vmxnet3_stat_desc
vmxnet3_rq_driver_stats[] = {
	/* description,         offset */
	{ "  drv dropped rx total", offsetof(struct vmxnet3_rq_driver_stats,
					   drop_total) },
	{ "     err",            offsetof(struct vmxnet3_rq_driver_stats,
					drop_err) },
	{ "     fcs",            offsetof(struct vmxnet3_rq_driver_stats,
					drop_fcs) },
	{ "  rx buf alloc fail", offsetof(struct vmxnet3_rq_driver_stats,
					rx_buf_alloc_failure) },
};

/* gloabl stats maintained by the driver */
static const struct vmxnet3_stat_desc
vmxnet3_global_stats[] = {
	/* description,         offset */
	{ "tx timeout count",   offsetof(struct vmxnet3_adapter,
					 tx_timeout_count) }
};

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)

/* return stats got from adapter/netdev */

struct rtnl_link_stats64 *
vmxnet3_get_stats64(struct net_device *netdev,
		    struct rtnl_link_stats64 *stats)
{
	struct vmxnet3_adapter *adapter;
	struct vmxnet3_tq_driver_stats *drvTxStats;
	struct vmxnet3_rq_driver_stats *drvRxStats;
	struct UPT1_TxStats *devTxStats;
	struct UPT1_RxStats *devRxStats;
	unsigned long flags;
	int i;

adapter = compat_netdev_priv(netdev);

/* Collect the dev stats into the shared area */
	spin_lock_irqsave(&adapter->cmd_lock, flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);

for (i = 0; i < adapter->num_tx_queues; i++) {
		devTxStats = &adapter->tqd_start[i].stats;
		drvTxStats = &adapter->tx_queue[i].stats;
		stats->tx_packets += devTxStats->ucastPktsTxOK +
					devTxStats->mcastPktsTxOK +
					devTxStats->bcastPktsTxOK;
		stats->tx_bytes += devTxStats->ucastBytesTxOK +
				      devTxStats->mcastBytesTxOK +
				      devTxStats->bcastBytesTxOK;
		stats->tx_errors += devTxStats->pktsTxError;
		stats->tx_dropped += drvTxStats->drop_total;
	}

for (i = 0; i < adapter->num_rx_queues; i++) {
		devRxStats = &adapter->rqd_start[i].stats;
		drvRxStats = &adapter->rx_queue[i].stats;
		stats->rx_packets += devRxStats->ucastPktsRxOK +
					devRxStats->mcastPktsRxOK +
					devRxStats->bcastPktsRxOK;

stats->rx_bytes += devRxStats->ucastBytesRxOK +
				      devRxStats->mcastBytesRxOK +
				      devRxStats->bcastBytesRxOK;

stats->rx_errors += devRxStats->pktsRxError;
		stats->rx_dropped += drvRxStats->drop_total;
		stats->multicast +=  devRxStats->mcastPktsRxOK;
	}
	return stats;
}

#else

/* Returns pointer to net_device_stats struct in the adapter/netdev */
struct net_device_stats *
vmxnet3_get_stats(struct net_device *netdev)
{
	struct vmxnet3_adapter *adapter;
	struct vmxnet3_tq_driver_stats *drvTxStats;
	struct vmxnet3_rq_driver_stats *drvRxStats;
	struct UPT1_TxStats *devTxStats;
	struct UPT1_RxStats *devRxStats;
	struct net_device_stats *net_stats;
	unsigned long flags;
	int i;

adapter = compat_netdev_priv(netdev);
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 22)
	net_stats = &adapter->net_stats;
#else
	net_stats = &netdev->stats;
#endif

memset(net_stats, 0, sizeof(*net_stats));
	for (i = 0; i < adapter->num_tx_queues; i++) {
		devTxStats = &adapter->tqd_start[i].stats;
		drvTxStats = &adapter->tx_queue[i].stats;
		net_stats->tx_packets += devTxStats->ucastPktsTxOK +
					devTxStats->mcastPktsTxOK +
					devTxStats->bcastPktsTxOK;
		net_stats->tx_bytes += devTxStats->ucastBytesTxOK +
				      devTxStats->mcastBytesTxOK +
				      devTxStats->bcastBytesTxOK;
		net_stats->tx_errors += devTxStats->pktsTxError;
		net_stats->tx_dropped += drvTxStats->drop_total;
	}

for (i = 0; i < adapter->num_rx_queues; i++) {
		devRxStats = &adapter->rqd_start[i].stats;
		drvRxStats = &adapter->rx_queue[i].stats;
		net_stats->rx_packets += devRxStats->ucastPktsRxOK +
					devRxStats->mcastPktsRxOK +
					devRxStats->bcastPktsRxOK;

net_stats->rx_bytes += devRxStats->ucastBytesRxOK +
				      devRxStats->mcastBytesRxOK +
				      devRxStats->bcastBytesRxOK;

net_stats->rx_errors += devRxStats->pktsRxError;
		net_stats->rx_dropped += drvRxStats->drop_total;
		net_stats->multicast +=  devRxStats->mcastPktsRxOK;
	}
	return net_stats;
}

#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)

/* Returns the number of counters we will return in vmxnet3_get_ethtool_stats.
 * Assume each counter is uint64.
 */
static int
vmxnet3_get_stats_count(struct net_device *netdev)
{
	return  ARRAY_SIZE(vmxnet3_tq_dev_stats) +
		ARRAY_SIZE(vmxnet3_tq_driver_stats) +
		ARRAY_SIZE(vmxnet3_rq_dev_stats) +
		ARRAY_SIZE(vmxnet3_rq_driver_stats) +
		ARRAY_SIZE(vmxnet3_global_stats);
}

#else

static int
vmxnet3_get_sset_count(struct net_device *netdev, int sset)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	switch (sset) {
	case ETH_SS_STATS:
		return  (ARRAY_SIZE(vmxnet3_tq_dev_stats) +
			 ARRAY_SIZE(vmxnet3_tq_driver_stats)) *
			adapter->num_tx_queues +
			(ARRAY_SIZE(vmxnet3_rq_dev_stats) +
			 ARRAY_SIZE(vmxnet3_rq_driver_stats)) *
			adapter->num_rx_queues +
			ARRAY_SIZE(vmxnet3_global_stats);
	default:
		return -EOPNOTSUPP;
	}
}

#endif

/* Should be multiple of 4 */
#define NUM_TX_REGS	8
#define NUM_RX_REGS	12

static int
vmxnet3_get_regs_len(struct net_device *netdev)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	return (adapter->num_tx_queues * NUM_TX_REGS * sizeof(u32) +
		adapter->num_rx_queues * NUM_RX_REGS * sizeof(u32));
}

/*
 * *drvinfo is updated
 */
static void
vmxnet3_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);

strlcpy(drvinfo->driver, vmxnet3_driver_name, sizeof(drvinfo->driver));

strlcpy(drvinfo->version, VMXNET3_DRIVER_VERSION_REPORT,
		sizeof(drvinfo->version));

#if COMPAT_LINUX_VERSION_CHECK_LT(3, 1, 0)
	strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version));
#endif

strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
		ETHTOOL_BUSINFO_LEN);

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
	drvinfo->n_stats = vmxnet3_get_sset_count(netdev, ETH_SS_STATS);
#else
	drvinfo->n_stats = vmxnet3_get_stats_count(netdev);
#endif
	drvinfo->testinfo_len = 0;
	drvinfo->eedump_len   = 0;
	drvinfo->regdump_len  = vmxnet3_get_regs_len(netdev);
}

/* Returns the description strings for the counters returned by
 * vmxnet3_get_ethtool_stats.
 */
static void
vmxnet3_get_strings(struct net_device *netdev, u32 stringset, u8 *buf)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	if (stringset == ETH_SS_STATS) {
		int i, j;
		for (j = 0; j < adapter->num_tx_queues; j++) {
			for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_dev_stats); i++) {
				memcpy(buf, vmxnet3_tq_dev_stats[i].desc,
				       ETH_GSTRING_LEN);
				buf += ETH_GSTRING_LEN;
			}
			for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_driver_stats); i++) {
				memcpy(buf, vmxnet3_tq_driver_stats[i].desc,
				       ETH_GSTRING_LEN);
				buf += ETH_GSTRING_LEN;
			}
		}

for (j = 0; j < adapter->num_rx_queues; j++) {
			for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_dev_stats); i++) {
				memcpy(buf, vmxnet3_rq_dev_stats[i].desc,
				       ETH_GSTRING_LEN);
				buf += ETH_GSTRING_LEN;
			}
			for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_driver_stats); i++) {
				memcpy(buf, vmxnet3_rq_driver_stats[i].desc,
				       ETH_GSTRING_LEN);
				buf += ETH_GSTRING_LEN;
			}
		}

for (i = 0; i < ARRAY_SIZE(vmxnet3_global_stats); i++) {
			memcpy(buf, vmxnet3_global_stats[i].desc,
				ETH_GSTRING_LEN);
			buf += ETH_GSTRING_LEN;
		}
	}
}

#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 23)

#  if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
int vmxnet3_set_features(struct net_device *netdev, compat_netdev_features_t features) {
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	unsigned long flags;
	compat_netdev_features_t changed = features ^ netdev->features;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0)
	if (changed & (NETIF_F_RXCSUM | NETIF_F_LRO | NETIF_F_HW_VLAN_CTAG_RX)) {
#else
	if (changed & (NETIF_F_RXCSUM | NETIF_F_LRO | NETIF_F_HW_VLAN_RX)) {
#endif
		if (features & NETIF_F_RXCSUM) {
			adapter->shared->devRead.misc.uptFeatures |=
				UPT1_F_RXCSUM;
			adapter->rxcsum = TRUE;
		} else {
			adapter->shared->devRead.misc.uptFeatures &=
				~UPT1_F_RXCSUM;
			adapter->rxcsum = FALSE;
		}

if (features & NETIF_F_LRO) {
			adapter->shared->devRead.misc.uptFeatures |=
				UPT1_F_LRO;
			adapter->lro = TRUE;
		} else {
			adapter->shared->devRead.misc.uptFeatures &=
				~UPT1_F_LRO;
			adapter->lro = FALSE;
		}

#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0)
		if (features & NETIF_F_HW_VLAN_CTAG_RX)
#else
		if (features & NETIF_F_HW_VLAN_RX)
#endif
			adapter->shared->devRead.misc.uptFeatures |=
				UPT1_F_RXVLAN;
		else
			adapter->shared->devRead.misc.uptFeatures &=
				~UPT1_F_RXVLAN;

spin_lock_irqsave(&adapter->cmd_lock, flags);
		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
				VMXNET3_CMD_UPDATE_FEATURE);
		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
	}
	return 0;
}

#  else

static u32
vmxnet3_get_flags(struct net_device *netdev) {
	return netdev->features;
}

static int
vmxnet3_set_flags(struct net_device *netdev, u32 data) {
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	u8 lro_requested = (data & ETH_FLAG_LRO) == 0 ? 0 : 1;
	u8 lro_present = (netdev->features & NETIF_F_LRO) == 0 ? 0 : 1;
	unsigned long flags;

if (lro_requested != lro_present) {
		/* toggle the LRO feature*/
		netdev->features ^= NETIF_F_LRO;
		adapter->lro = lro_requested;

/* update harware LRO capability accordingly */
		if (lro_requested)
			adapter->shared->devRead.misc.uptFeatures |= UPT1_F_LRO;
		else
			adapter->shared->devRead.misc.uptFeatures &=
								~UPT1_F_LRO;
		spin_lock_irqsave(&adapter->cmd_lock, flags);
		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
				       VMXNET3_CMD_UPDATE_FEATURE);
		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
	}
	return 0;
}
#  endif
#endif

static void
vmxnet3_get_ethtool_stats(struct net_device *netdev,
			  struct ethtool_stats *stats, u64 *buf)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	unsigned long flags;
	u8 *base;
	int i;
	int j = 0;

spin_lock_irqsave(&adapter->cmd_lock, flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);

/* this does assume each counter is 64-bit wide */
	for (j = 0; j < adapter->num_tx_queues; j++) {
		base = (u8 *)&adapter->tqd_start[j].stats;
		*buf++ = (u64)j;
		for (i = 1; i < ARRAY_SIZE(vmxnet3_tq_dev_stats); i++)
			*buf++ = *(u64 *)(base + vmxnet3_tq_dev_stats[i].offset);

base = (u8 *)&adapter->tx_queue[j].stats;
		for (i = 0; i < ARRAY_SIZE(vmxnet3_tq_driver_stats); i++)
			*buf++ = *(u64 *)(base + vmxnet3_tq_driver_stats[i].offset);
	}

for (j = 0; j < adapter->num_tx_queues; j++) {
		base = (u8 *)&adapter->rqd_start[j].stats;
		*buf++ = (u64) j;
		for (i = 1; i < ARRAY_SIZE(vmxnet3_rq_dev_stats); i++)
			*buf++ = *(u64 *)(base + vmxnet3_rq_dev_stats[i].offset);

base = (u8 *)&adapter->rx_queue[j].stats;
		for (i = 0; i < ARRAY_SIZE(vmxnet3_rq_driver_stats); i++)
			*buf++ = *(u64 *)(base + vmxnet3_rq_driver_stats[i].offset);
	}

base = (u8 *)adapter;
	for (i = 0; i < ARRAY_SIZE(vmxnet3_global_stats); i++)
		*buf++ = *(u64 *)(base + vmxnet3_global_stats[i].offset);
}

static void
vmxnet3_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	u32 *buf = p;
	int i = 0, j = 0;

memset(p, 0, vmxnet3_get_regs_len(netdev));

regs->version = 1;

/* Update vmxnet3_get_regs_len if we want to dump more registers */

/* make each ring use multiple of 16 bytes */
	for (i = 0; i < adapter->num_tx_queues; i++) {
		buf[j++] = adapter->tx_queue[i].tx_ring.next2fill;
		buf[j++] = adapter->tx_queue[i].tx_ring.next2comp;
		buf[j++] = adapter->tx_queue[i].tx_ring.gen;
		buf[j++] = 0;

buf[j++] = adapter->tx_queue[i].comp_ring.next2proc;
		buf[j++] = adapter->tx_queue[i].comp_ring.gen;
		buf[j++] = adapter->tx_queue[i].stopped;
		buf[j++] = 0;
	}

for (i = 0; i < adapter->num_rx_queues; i++) {
		buf[j++] = adapter->rx_queue[i].rx_ring[0].next2fill;
		buf[j++] = adapter->rx_queue[i].rx_ring[0].next2comp;
		buf[j++] = adapter->rx_queue[i].rx_ring[0].gen;
		buf[j++] = 0;

buf[j++] = adapter->rx_queue[i].rx_ring[1].next2fill;
		buf[j++] = adapter->rx_queue[i].rx_ring[1].next2comp;
		buf[j++] = adapter->rx_queue[i].rx_ring[1].gen;
		buf[j++] = 0;

buf[j++] = adapter->rx_queue[i].comp_ring.next2proc;
		buf[j++] = adapter->rx_queue[i].comp_ring.gen;
		buf[j++] = 0;
		buf[j++] = 0;
	}

}

static void
vmxnet3_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);

wol->supported = WAKE_UCAST | WAKE_ARP | WAKE_MAGIC;
	wol->wolopts = adapter->wol;
}

static int
vmxnet3_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);

if (wol->wolopts & (WAKE_PHY | WAKE_MCAST | WAKE_BCAST |
			    WAKE_MAGICSECURE)) {
		return -EOPNOTSUPP;
	}

adapter->wol = wol->wolopts;

compat_device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);

return 0;
}

static int
vmxnet3_get_settings(struct net_device *netdev, struct ethtool_cmd *ecmd)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);

ecmd->supported = SUPPORTED_10000baseT_Full | SUPPORTED_1000baseT_Full |
			  SUPPORTED_TP;
	ecmd->advertising = ADVERTISED_TP;
	ecmd->port = PORT_TP;
	ecmd->transceiver = XCVR_INTERNAL;

if (adapter->link_speed) {
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 39)
		ecmd->speed = adapter->link_speed;
#else
		ethtool_cmd_speed_set(ecmd, adapter->link_speed);
#endif
		ecmd->duplex = DUPLEX_FULL;
	} else {
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 39)
		ecmd->speed = -1;
#else
		ethtool_cmd_speed_set(ecmd, -1);
#endif
		ecmd->duplex = -1;
	}
	return 0;
}

static void
vmxnet3_get_ringparam(struct net_device *netdev,
		      struct ethtool_ringparam *param)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);

param->rx_max_pending = VMXNET3_RX_RING_MAX_SIZE;
	param->tx_max_pending = VMXNET3_TX_RING_MAX_SIZE;
	param->rx_mini_max_pending = 0;
	param->rx_jumbo_max_pending = VMXNET3_RX_RING2_MAX_SIZE;

param->rx_pending = adapter->rx_queue[0].rx_ring[0].size;
	param->tx_pending = adapter->tx_queue[0].tx_ring.size;
	param->rx_mini_pending = 0;
	param->rx_jumbo_pending = adapter->rx_queue[0].rx_ring[1].size;
}

static int
vmxnet3_set_ringparam(struct net_device *netdev,
		      struct ethtool_ringparam *param)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	u32 new_tx_ring_size, new_rx_ring_size, new_rx_ring2_size;
	u32 sz;

if (param->tx_pending == 0 ||
	    param->tx_pending >	VMXNET3_TX_RING_MAX_SIZE)
		return -EINVAL;

if (param->rx_pending == 0 ||
	    param->rx_pending >	VMXNET3_RX_RING_MAX_SIZE)
		return -EINVAL;

if (param->rx_jumbo_pending == 0 ||
            param->rx_jumbo_pending > VMXNET3_RX_RING2_MAX_SIZE)
                return -EINVAL;

/* if adapter not yet initialized, do nothing */
	if (adapter->rx_buf_per_pkt == 0) {
		printk(KERN_INFO "%s: adapter not completely initialized, "
		       "ring size cannot be changed yet.\n", netdev->name);
		return -EOPNOTSUPP;
	}

/* round it up to a multiple of VMXNET3_RING_SIZE_ALIGN */
	new_tx_ring_size = (param->tx_pending + VMXNET3_RING_SIZE_MASK) &
		~VMXNET3_RING_SIZE_MASK;
	new_tx_ring_size = min_t(u32, new_tx_ring_size,
				 VMXNET3_TX_RING_MAX_SIZE);
	BUG_ON(new_tx_ring_size > VMXNET3_TX_RING_MAX_SIZE);
	BUG_ON(new_tx_ring_size % VMXNET3_RING_SIZE_ALIGN != 0);

/* ring0 has to be a multiple of
	 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
	 */
	sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
	new_rx_ring_size = (param->rx_pending + sz - 1) / sz * sz;
	new_rx_ring_size = min_t(u32, new_rx_ring_size,
			VMXNET3_RX_RING_MAX_SIZE / sz * sz);
	BUG_ON(new_rx_ring_size > VMXNET3_RX_RING_MAX_SIZE);
	BUG_ON(new_rx_ring_size % sz != 0);

/* ring1 has to be a multiple of VMXNET3_RING_SIZE_ALIGN */
        new_rx_ring2_size = (param->rx_jumbo_pending + VMXNET3_RING_SIZE_MASK) &
                ~VMXNET3_RING_SIZE_MASK;
        new_rx_ring2_size = min_t(u32, new_rx_ring2_size,
                                  VMXNET3_RX_RING2_MAX_SIZE);
        BUG_ON(new_rx_ring2_size > VMXNET3_RX_RING2_MAX_SIZE);
        BUG_ON(new_rx_ring2_size % VMXNET3_RING_SIZE_ALIGN != 0);

if(adapter->use_adaptive_ring) {
		adapter->use_adaptive_ring = FALSE;
		printk(KERN_INFO "%s: User specified custom ring size, "
		       "disabling adaptive ring support.\n", netdev->name);
	}

return vmxnet3_set_ringsize(netdev, new_tx_ring_size, new_rx_ring_size,
                                    new_rx_ring2_size);
}

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)
static int
vmxnet3_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info,
#   if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
                  void *rules)
#   else
		  u32 *rules)
#   endif
{
	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
	switch (info->cmd) {
	case ETHTOOL_GRXRINGS:
		info->data = adapter->num_rx_queues;
		return 0;
	}
	return -EOPNOTSUPP;
}
#endif

#if defined(VMXNET3_RSS) && LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)

#   if COMPAT_LINUX_VERSION_CHECK_LT(3, 3, 0) && !defined(ORACLE_UEK2_300_VMXNET3_FIX)
#   else
static u32
vmxnet3_get_rss_indir_size(struct net_device *netdev)
{
	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
	struct UPT1_RSSConf *rssConf = adapter->rss_conf;

return rssConf->indTableSize;
}

#   endif

static int
#   if COMPAT_LINUX_VERSION_CHECK_LT(3, 3, 0) && !defined(ORACLE_UEK2_300_VMXNET3_FIX)
vmxnet3_get_rss_indir(struct net_device *netdev,
		      struct ethtool_rxfh_indir *p)
#   else
vmxnet3_get_rss_indir(struct net_device *netdev, u32 *p)
#   endif
{
	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
	struct UPT1_RSSConf *rssConf = adapter->rss_conf;
#   if COMPAT_LINUX_VERSION_CHECK_LT(3, 3, 0) && !defined(ORACLE_UEK2_300_VMXNET3_FIX)
	unsigned int n = min_t(unsigned int, p->size, rssConf->indTableSize);
	p->size = rssConf->indTableSize;
	while (n--)
		p->ring_index[n] = rssConf->indTable[n];
#   else
	unsigned int n = rssConf->indTableSize;
	BUG_ON(!n);
	while (n--)
		p[n] = rssConf->indTable[n];
#   endif
	return 0;
}

static int
#   if COMPAT_LINUX_VERSION_CHECK_LT(3, 3, 0) && !defined(ORACLE_UEK2_300_VMXNET3_FIX)
vmxnet3_set_rss_indir(struct net_device *netdev,
		       const struct ethtool_rxfh_indir *p)
#   else
vmxnet3_set_rss_indir(struct net_device *netdev, const u32 *p)
#   endif
 {
        unsigned int i;
        unsigned long flags;
        struct vmxnet3_adapter *adapter = netdev_priv(netdev);
        struct UPT1_RSSConf *rssConf = adapter->rss_conf;
#   if COMPAT_LINUX_VERSION_CHECK_LT(3, 3, 0) && !defined(ORACLE_UEK2_300_VMXNET3_FIX)
	const u32 *idxArray = p->ring_index;
	if (p->size != rssConf->indTableSize)
		return -EINVAL;
#   else
	const u32 *idxArray = p;
#   endif

for (i = 0; i < rssConf->indTableSize; i++) {
		/*
		 * Return with error code if any of the queue indices
		 * is out of range
		 */
		if (idxArray[i] >= adapter->num_rx_queues)
			return -EINVAL;
	}

for (i = 0; i < rssConf->indTableSize; i++)
		rssConf->indTable[i] = idxArray[i];

spin_lock_irqsave(&adapter->cmd_lock, flags);
        VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
			       VMXNET3_CMD_UPDATE_RSSIDT);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);

return 0;

}

#endif

static const struct ethtool_ops vmxnet3_ethtool_ops = {
	.get_settings      = vmxnet3_get_settings,
	.get_drvinfo       = vmxnet3_get_drvinfo,
	.get_regs_len      = vmxnet3_get_regs_len,
	.get_regs          = vmxnet3_get_regs,
	.get_wol           = vmxnet3_get_wol,
	.set_wol           = vmxnet3_set_wol,
	.get_link          = ethtool_op_get_link,
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 39)
	.get_rx_csum       = vmxnet3_get_rx_csum,
	.set_rx_csum       = vmxnet3_set_rx_csum,
	.get_tx_csum       = vmxnet3_get_tx_csum,
	.set_tx_csum       = vmxnet3_set_tx_csum,
	.get_sg            = ethtool_op_get_sg,
	.set_sg            = ethtool_op_set_sg,
	.get_tso           = ethtool_op_get_tso,
	.set_tso           = vmxnet3_set_tso,
#  if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
	.get_flags	   = vmxnet3_get_flags,
	.set_flags	   = vmxnet3_set_flags,
#  endif
#endif
	.get_strings       = vmxnet3_get_strings,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
	.get_sset_count	   = vmxnet3_get_sset_count,
#else
	.get_stats_count   = vmxnet3_get_stats_count,
#endif
	.get_ethtool_stats = vmxnet3_get_ethtool_stats,
	.get_ringparam     = vmxnet3_get_ringparam,
	.set_ringparam     = vmxnet3_set_ringparam,
#if defined(ETHTOOL_GPERMADDR) && (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23)) && (LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 13))
	.get_perm_addr     = ethtool_op_get_perm_addr,
#endif
#if  LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)
	.get_rxnfc         = vmxnet3_get_rxnfc,
#   ifdef VMXNET3_RSS
	.get_rxfh_indir    = vmxnet3_get_rss_indir,
	.set_rxfh_indir    = vmxnet3_set_rss_indir,
#      if COMPAT_LINUX_VERSION_CHECK_LT(3, 3, 0) && !defined(ORACLE_UEK2_300_VMXNET3_FIX)
#      else
	.get_rxfh_indir_size = vmxnet3_get_rss_indir_size,
#      endif
#   endif
#endif
};

void vmxnet3_set_ethtool_ops(struct net_device *netdev)
{
#   if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
	SET_ETHTOOL_OPS(netdev, &vmxnet3_ethtool_ops);
#   else
	/* Need to typecase to avoid the warning for const */
	SET_ETHTOOL_OPS(netdev, (struct ethtool_ops *) &vmxnet3_ethtool_ops);
#   endif
}

��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/vmxnet2_def.h��������������������������������������������������������������������������0000444�0000000�0000000�00000033643�13432725350�014544� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2004-2014,2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef _VMXNET2_DEF_H_
#define _VMXNET2_DEF_H_

#define INCLUDE_ALLOW_USERLEVEL

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#include "includeCheck.h"

#include "net_sg.h"
#include "vmxnet_def.h"

#if defined __cplusplus
extern "C" {
#endif

/*
 * Magic number that identifies this version of the vmxnet protocol.
 */
#define VMXNET2_MAGIC			0xbabe864f

/* size of the rx ring */
#define VMXNET2_MAX_NUM_RX_BUFFERS		128
#define VMXNET2_DEFAULT_NUM_RX_BUFFERS	        100

/* size of the rx ring when enhanced vmxnet is used */
#define ENHANCED_VMXNET2_MAX_NUM_RX_BUFFERS     512 
#define ENHANCED_VMXNET2_DEFAULT_NUM_RX_BUFFERS 150

/* size of the 2nd rx ring */
#define VMXNET2_MAX_NUM_RX_BUFFERS2             2048 
#define VMXNET2_DEFAULT_NUM_RX_BUFFERS2	        512

/* size of the tx ring */
#define VMXNET2_MAX_NUM_TX_BUFFERS		128
#define VMXNET2_DEFAULT_NUM_TX_BUFFERS	        100

/* size of the tx ring when tso/jf is used */
#define VMXNET2_MAX_NUM_TX_BUFFERS_TSO          512
#define VMXNET2_DEFAULT_NUM_TX_BUFFERS_TSO	256

enum {
   VMXNET2_OWNERSHIP_DRIVER,
   VMXNET2_OWNERSHIP_DRIVER_PENDING,
   VMXNET2_OWNERSHIP_NIC,
   VMXNET2_OWNERSHIP_NIC_PENDING,
   VMXNET2_OWNERSHIP_NIC_FRAG,
   VMXNET2_OWNERSHIP_DRIVER_FRAG,
};

#define VMXNET2_SG_DEFAULT_LENGTH	6

typedef struct Vmxnet2_SG_Array {
   uint16	addrType;
   uint16	length;
   NetSG_Elem	sg[VMXNET2_SG_DEFAULT_LENGTH];
} Vmxnet2_SG_Array;

typedef struct Vmxnet2_RxRingEntry {
   uint64		paddr;		/* Physical address of the packet data. */
   uint32		bufferLength;	/* The length of the data at paddr. */
   uint32		actualLength;	/* The actual length of the received data. */
   uint16               ownership;	/* Who owns the packet. */
   uint16		flags;		/* Flags as defined below. */
   uint32               index;          /* 
                                         * Currently:
                                         *
                                         * This is being used as an packet index to
                                         * rx buffers.
                                         *
                                         * Originally: 
                                         *
					 * was void* driverData ("Driver specific data.")
					 * which was used for sk_buf**s in Linux and
                                         * VmxnetRxBuff*s in Windows.  It could not be
					 * here because the structure needs to be the
					 * same size between architectures, and it was
					 * not used on the device side, anyway.  Look
					 * for its replacement in
					 * Vmxnet_Private.rxRingBuffPtr on Linux and
					 * VmxnetAdapter.rxRingBuffPtr on Windows.
					 */
} Vmxnet2_RxRingEntry;

/*
 * Vmxnet2_RxRingEntry flags:
 * 
 * VMXNET2_RX_HW_XSUM_OK       The hardware verified the TCP/UDP checksum.
 * VMXNET2_RX_WITH_FRAG        More data is in the 2nd ring
 * VMXNET2_RX_FRAG_EOP         This is the last frag, the only valid flag for
 *                             2nd ring entry
 *
 */
#define VMXNET2_RX_HW_XSUM_OK  0x01
#define VMXNET2_RX_WITH_FRAG   0x02
#define VMXNET2_RX_FRAG_EOP    0x04

typedef struct Vmxnet2_TxRingEntry {
   uint16		flags;		/* Flags as defined below. */
   uint16 	        ownership;	/* Who owns this packet. */
   uint32               extra;          /*
					 * was void* driverData ("Driver specific data.")
					 * which was used for sk_buf*s in Linux and
                                         * VmxnetTxInfo*s in Windows.  It could not be
					 * here because the structure needs to be the
					 * same size between architectures, and it was
					 * not used on the device side, anyway.  Look
					 * for its replacement in
					 * Vmxnet_Private.txRingBuffPtr on Linux and
					 * VmxnetAdapter.txRingBuffPtr on Windows.
					 */
   uint32               tsoMss;         /* TSO pkt MSS */
   Vmxnet2_SG_Array	sg;		/* Packet data. */
} Vmxnet2_TxRingEntry;

/*
 * Vmxnet2_TxRingEntry flags:
 *
 *   VMXNET2_TX_CAN_KEEP	The implementation can return the tx ring entry 
 *				to the driver when it is ready as opposed to 
 *				before the transmit call from the driver completes.
 *   VMXNET2_TX_RING_LOW	The driver's transmit ring buffer is low on free
 *				slots.
 *   VMXNET2_TX_HW_XSUM         The hardware should perform the TCP/UDP checksum
 *   VMXNET2_TX_TSO             The hardware should do TCP segmentation.
 *   VMXNET2_TX_PINNED_BUFFER   The driver used one of the preallocated vmkernel
 *                              buffers *and* it has been pinned with Net_PinTxBuffers.
 *   VMXNET2_TX_MORE            This is *not* the last tx entry for the pkt.
 *                              All flags except VMXNET2_TX_MORE are ignored
 *                              for the subsequent tx entries.
 */
#define VMXNET2_TX_CAN_KEEP	     0x0001
#define VMXNET2_TX_RING_LOW	     0x0002
#define VMXNET2_TX_HW_XSUM           0x0004
#define VMXNET2_TX_TSO	             0x0008
#define VMXNET2_TX_PINNED_BUFFER     0x0010
#define VMXNET2_TX_MORE              0x0020

/*
 * Structure used by implementations.  This structure allows the inline
 * functions below to be used.
 */
typedef struct Vmxnet2_RxRingInfo {
#ifndef VMX86_VMX
   Vmxnet2_RxRingEntry    *base;       /* starting addr of the ring */
#endif
   uint32                  nicNext;    /* next entry to use in the ring */
   uint32                  ringLength; /* # of entries in the ring */
   PA                      startPA;    /* starting addr of the ring */
#ifdef VMX86_DEBUG
   const char             *name;
#endif
} Vmxnet2_RxRingInfo;

typedef struct Vmxnet2_TxRingInfo {
#ifndef VMX86_VMX
   Vmxnet2_TxRingEntry    *base;       /* starting addr of the ring */
#endif
   uint32                  nicNext;    /* next entry to use in the ring */
   uint32                  ringLength; /* # of entries in the ring */
   PA                      startPA;    /* starting addr of the ring */
#ifdef VMX86_DEBUG
   const char             *name;
#endif
} Vmxnet2_TxRingInfo;

typedef struct Vmxnet2_ImplData {
   Vmxnet2_RxRingInfo    rxRing;
   Vmxnet2_RxRingInfo    rxRing2;
   Vmxnet2_TxRingInfo    txRing;
} Vmxnet2_ImplData;

typedef struct Vmxnet2_DriverStats {
   uint32	transmits;	   /* # of times that the drivers transmit function */
				   /*   is called. The driver could transmit more */
				   /*   than one packet per call. */
   uint32	pktsTransmitted;   /* # of packets transmitted. */
   uint32	noCopyTransmits;   /* # of packets that are transmitted without */
				   /*   copying any data. */
   uint32	copyTransmits;	   /* # of packets that are transmittted by copying */
				   /*   the data into a buffer. */
   uint32	maxTxsPending;	   /* Max # of transmits outstanding. */
   uint32	txStopped;	   /* # of times that transmits got stopped because */
				   /*   the tx ring was full. */
   uint32	txRingOverflow;	   /* # of times that transmits got deferred bc */
				   /*   the tx ring was full.  This must be >= */
				   /*   txStopped since there will be one */
				   /*   txStopped when the ring fills up and then */
				   /*   one txsRingOverflow for each packet that */
				   /*   that gets deferred until there is space. */
   uint32	interrupts;	   /* # of times interrupted. */
   uint32	pktsReceived;	   /* # of packets received. */
   uint32	rxBuffersLow;	   /* # of times that the driver was low on */
} Vmxnet2_DriverStats;

/*
 * Shared data structure between the vm, the vmm, and the vmkernel.
 * This structure was originally arranged to try to group common data 
 * on 32-byte cache lines, but bit rot and the fact that we no longer
 * run on many CPUs with that cacheline size killed that optimization.
 * vmxnet3 should target 128 byte sizes and alignments to optimize for
 * the 64 byte cacheline pairs on P4.
 */
typedef struct Vmxnet2_DriverData {
   /*
    * Magic must be first.
    */
   Vmxnet_DDMagic       magic;

/*
    * Receive fields. 
    */
   uint32		rxRingLength;		/* Length of the receive ring. */
   uint32		rxDriverNext;		/* Index of the next packet that will */
						/*   be filled in by the impl */

uint32		rxRingLength2;	        /* Length of the 2nd receive ring. */
   uint32		rxDriverNext2;	        /* Index of the next packet that will */
						/*   be filled in by the impl */

uint32		notUsed1;               /* was "irq" */

/*
    * Interface flags and multicast filter.
    */
   uint32		ifflags;
   uint32		LADRF[VMXNET_MAX_LADRF];

/*
    * Transmit fields
    */
   uint32               txDontClusterSize;      /* All packets <= this will be transmitted */
                                                /* immediately, regardless of clustering */
                                                /* settings [was fill[1]] */
   uint32		txRingLength;		/* Length of the transmit ring. */
   uint32		txDriverCur;		/* Index of the next packet to be */
						/*   returned by the implementation.*/
   uint32		txDriverNext;		/* Index of the entry in the ring */
						/*   buffer to use for the next packet.*/
   uint32		txStopped;  		/* The driver has stopped transmitting */
						/*   because its ring buffer is full.*/
   uint32		txClusterLength;	/* Maximum number of packets to */
						/*   put in the ring buffer before */
						/*   asking the implementation to */
						/*   transmit the packets in the buffer.*/
   uint32		txNumDeferred;          /* Number of packets that have been */
						/*   queued in the ring buffer since */
						/*   the last time the implementation */
						/*   was asked to transmit. */
   uint32		notUsed3;               /* This field is deprecated but still used */
                                                /* as minXmitPhysLength on the escher branch. */
                                                /* It cannot be used for other purposes */
                                                /* until escher vms no longer are allowed */
                                                /* to install this driver. */

uint32              totalRxBuffers;          /* used by esx for max rx buffers */
   uint64              rxBufferPhysStart;       /* used by esx for pinng rx buffers */
   /*
    * Extra fields for future expansion.
    */
   uint32		extra[2];

uint16               maxFrags;               /* # of frags the driver can handle */
   uint16               featureCtl;             /* for driver to enable some feature */

/*
    * The following fields are used to save the nicNext indexes part
    * of implData in the vmkernel when disconnecting the adapter, we
    * need them when we reconnect.  This mechanism is used for
    * checkpointing as well.
    */
   uint32               savedRxNICNext;
   uint32               savedRxNICNext2;
   uint32               savedTxNICNext;

/*
    * Fields used during initialization or debugging.
    */
   uint32		length;
   uint32		rxRingOffset;
   uint32		rxRingOffset2;
   uint32		txRingOffset;   
   uint32		debugLevel;
   uint32		txBufferPhysStart;
   uint32		txBufferPhysLength;
   uint32		txPktMaxSize;

/*
    * Driver statistics.
    */
   Vmxnet2_DriverStats	stats;
} Vmxnet2_DriverData;

#ifdef VMX86_SERVER
/* 
 * Shared between VMM and Vmkernel part of vmxnet2 to optimize action posting
 * VMM writes 1 (don't post) or 0 (okay to post) and vmk reads this.
 */
typedef struct VmxnetVMKShared {
   uint32    dontPostActions;
   PciHandle pciHandle;
} VmxnetVMKShared;
#endif

#if defined VMKERNEL

/*
 * Inline functions used to assist the implementation of the vmxnet interface.
 */

/*
 * Get the next empty packet out of the receive ring and move to 
 * the next packet.
 */
static INLINE Vmxnet2_RxRingEntry *
Vmxnet2_GetNextRx(Vmxnet2_RxRingInfo *ri, uint16 ownership)
{
   Vmxnet2_RxRingEntry *rre = ri->base + ri->nicNext;
   if (rre->ownership == ownership) {
      VMXNET_INC(ri->nicNext, ri->ringLength);
   } else {
      rre = NULL;
   }

return rre;
}

/*
 * Return ownership of a packet in the receive ring to the driver.
 */
static INLINE void
Vmxnet2_PutRx(Vmxnet2_RxRingEntry *rre, uint32 pktLength, uint16 ownership)
{
   rre->actualLength = pktLength;
   COMPILER_MEM_BARRIER();
   rre->ownership = ownership;
}

/*
 * Get the next pending packet out of the transmit ring.
 */
static INLINE Vmxnet2_TxRingEntry *
Vmxnet2_GetNextTx(Vmxnet2_TxRingInfo *ri)
{
   Vmxnet2_TxRingEntry *txre = ri->base + ri->nicNext;
   if (txre->ownership == VMXNET2_OWNERSHIP_NIC) {
      return txre;
   } else {
      return NULL;
   }
}

/*
 * Move to the next entry in the transmit ring.
 */
static INLINE unsigned int
Vmxnet2_IncNextTx(Vmxnet2_TxRingInfo *ri)
{
   unsigned int prev = ri->nicNext;
   Vmxnet2_TxRingEntry *txre = ri->base + ri->nicNext;
   
   txre->ownership = VMXNET2_OWNERSHIP_NIC_PENDING;

VMXNET_INC(ri->nicNext, ri->ringLength);
   return prev;
}

/*
 * Get the indicated entry from transmit ring.
 */
static INLINE Vmxnet2_TxRingEntry *
Vmxnet2_GetTxEntry(Vmxnet2_TxRingInfo *ri, unsigned int idx)
{
   return ri->base + idx;
}

/*
 * Get the indicated entry from the given rx ring
 */
static INLINE Vmxnet2_RxRingEntry *
Vmxnet2_GetRxEntry(Vmxnet2_RxRingInfo *ri, unsigned int idx)
{
   return ri->base + idx;
}

#endif /* defined VMX86_VMX || defined VMKERNEL */

#if defined __cplusplus
} // extern "C"
#endif

#endif // _VMXNET2_DEF_H_
���������������������������������������������������������������������������������������������vmxnet3-only/vmnet_def.h����������������������������������������������������������������������������0000444�0000000�0000000�00000017435�13432725350�014273� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2004-2014,2017-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmnet_def.h 
 *
 *     - definitions which are (mostly) not vmxnet or vlance specific
 */

#ifndef _VMNET_DEF_H_
#define _VMNET_DEF_H_

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_VMCORE

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#include "includeCheck.h"

#define VMNET_NAME_BUFFER_LEN  128 /* Increased for i18n. */
#define VMNET_COAL_STRING_LEN  128

/*
 * capabilities - not all of these are implemented in the virtual HW
 *                (eg VLAN support is in the virtual switch)  so even vlance
 *                can use them
 */
#define VMNET_CAP_SG                   CONST64U(0x0001)             /* Can do scatter-gather transmits. */
#define VMNET_CAP_IP4_CSUM             CONST64U(0x0002)             /* Can checksum only TCP/UDP over IPv4. */
#define VMNET_CAP_HW_CSUM              CONST64U(0x0004)             /* Can checksum all packets. */
#define VMNET_CAP_HIGH_DMA             CONST64U(0x0008)             /* Can DMA to high memory. */
#define VMNET_CAP_TOE                  CONST64U(0x0010)             /* Supports TCP/IP offload. */
#define VMNET_CAP_TSO                  CONST64U(0x0020)             /* Supports TCP Segmentation offload */
#define VMNET_CAP_SW_TSO               CONST64U(0x0040)             /* Supports SW TCP Segmentation */
#define VMNET_CAP_VMXNET_APROM         CONST64U(0x0080)             /* Vmxnet APROM support */
#define VMNET_CAP_HW_TX_VLAN           CONST64U(0x0100)             /* Can we do VLAN tagging in HW */
#define VMNET_CAP_HW_RX_VLAN           CONST64U(0x0200)             /* Can we do VLAN untagging in HW */
#define VMNET_CAP_SW_VLAN              CONST64U(0x0400)             /* Can we do VLAN tagging/untagging in SW */
#define VMNET_CAP_WAKE_PCKT_RCV        CONST64U(0x0800)             /* Can wake on network packet recv? */
#define VMNET_CAP_ENABLE_INT_INLINE    CONST64U(0x1000)             /* Enable Interrupt Inline */
#define VMNET_CAP_ENABLE_HEADER_COPY   CONST64U(0x2000)             /* copy header for vmkernel */
#define VMNET_CAP_TX_CHAIN             CONST64U(0x4000)             /* Guest can use multiple tx entries for a pkt */
#define VMNET_CAP_RX_CHAIN             CONST64U(0x8000)             /* a pkt can span multiple rx entries */
#define VMNET_CAP_LPD                  CONST64U(0x10000)            /* large pkt delivery */
#define VMNET_CAP_BPF                  CONST64U(0x20000)            /* BPF Support in VMXNET Virtual Hardware */
#define VMNET_CAP_SG_SPAN_PAGES        CONST64U(0x40000)            /* Can do scatter-gather span multiple pages transmits. */
#define VMNET_CAP_IP6_CSUM             CONST64U(0x80000)            /* Can do IPv6 csum offload. */
#define VMNET_CAP_TSO6                 CONST64U(0x100000)           /* Can do TSO segmentation offload for IPv6 pkts. */
#define VMNET_CAP_TSO256k              CONST64U(0x200000)           /* Can do TSO segmentation offload for pkts up to 256kB. */
#define VMNET_CAP_UPT                  CONST64U(0x400000)           /* Support UPT */
#define VMNET_CAP_RDONLY_INETHDRS      CONST64U(0x800000)           /* Modifies inet headers for TSO/CSUm */
#define VMNET_CAP_ENCAP                CONST64U(0x1000000)          /* NPA not used, so redefining for ENCAP support */
#define VMNET_CAP_DCB                  CONST64U(0x2000000)          /* Support DCB */
#define VMNET_CAP_OFFSET_BASED_OFFLOAD CONST64U(0x4000000)       /* Support offload based offload */
#define VMNET_CAP_GENEVE_OFFLOAD       CONST64U(0x8000000)          /* Support Geneve encapsulation offload */
#define VMNET_CAP_IP6_CSUM_EXT_HDRS    CONST64U(0x10000000)         /* support csum of ip6 ext hdrs */
#define VMNET_CAP_TSO6_EXT_HDRS        CONST64U(0x20000000)         /* support TSO for ip6 ext hdrs */
#define VMNET_CAP_SCHED                CONST64U(0x40000000)         /* compliant with network scheduling */
#define VMNET_CAP_SRIOV                CONST64U(0x80000000)         /* Supports SR-IOV */

#define VMNET_CAP_SG_TX                VMNET_CAP_SG
#define VMNET_CAP_SG_RX                CONST64U(0x200000000)        /* Scatter-gather receive capability */
#define VMNET_CAP_PRIV_STATS           CONST64U(0x400000000)        /* Driver supports accessing private stats */
#define VMNET_CAP_LINK_STATUS_SET      CONST64U(0x800000000)        /* Driver supports changing link status */
#define VMNET_CAP_MAC_ADDR_SET         CONST64U(0x1000000000)       /* Driver supports changing the interface MAC address */
#define VMNET_CAP_COALESCE_PARAMS      CONST64U(0x2000000000)       /* Driver supports changing interrupt coalescing parameters */
#define VMNET_CAP_VLAN_FILTER          CONST64U(0x4000000000)       /* VLAN Filtering capability */
#define VMNET_CAP_WAKE_ON_LAN          CONST64U(0x8000000000)       /* Wake-On-LAN capability */
#define VMNET_CAP_NETWORK_DUMP         CONST64U(0x10000000000)      /* Network core dumping capability */
#define VMNET_CAP_MULTI_QUEUE          CONST64U(0x20000000000)      /* Multiple queue capability */
#define VMNET_CAP_EEPROM               CONST64U(0x40000000000)      /* EEPROM dump capability */
#define VMNET_CAP_REGDUMP              CONST64U(0x80000000000)      /* Register dump capability */
#define VMNET_CAP_SELF_TEST            CONST64U(0x100000000000)     /* Self-test capability */
#define VMNET_CAP_PAUSE_PARAMS         CONST64U(0x200000000000)     /* Pause frame parameter adjusting */
#define VMNET_CAP_RESTART_NEG          CONST64U(0x400000000000)     /* Ability to restart negotiation of link speed/duplexity */
#define VMNET_CAP_LRO                  CONST64U(0x800000000000)     /* Hardware supported LRO */
#define VMNET_CAP_OFFLOAD_ALIGN_ANY    CONST64U(0x1000000000000)    /* Nic requires no header alignment */
#define VMNET_CAP_GENERIC_OFFLOAD      CONST64U(0x2000000000000)    /* Generic hardware offloading (eg. vxlan encap offload and offset based offload) */
#define VMNET_CAP_CABLE_TYPE           CONST64U(0x4000000000000)    /* Uplink supports getting and setting cable type. */
#define VMNET_CAP_PHY_ADDRESS          CONST64U(0x8000000000000)    /* Uplink supports getting and setting PHY address. */
#define VMNET_CAP_TRANSCEIVER_TYPE     CONST64U(0x10000000000000)   /* Uplink supports getting and setting transceiver type. */
#define VMNET_CAP_MESSAGE_LEVEL        CONST64U(0x20000000000000)   /* Uplink supports getting and setting message level. */
#define VMNET_CAP_RING_PARAMS          CONST64U(0x40000000000000)   /* Support getting/setting RX/TX ring size parameters */
#define VMNET_CAP_ADVERTISE_MODES      CONST64U(0x80000000000000)   /* Support getting/setting interconnect modes */
#define VMNET_CAP_HW_DCB               CONST64U(0x100000000000000)  /* DCB capability in hardware */
#define VMNET_CAP_RX_SW_LRO            CONST64U(0x200000000000000)  /* Suppport SW LRO */
#define VMNET_CAP_ENS                  CONST64U(0x400000000000000)  /* Suppport ENS */
#define VMNET_CAP_LEGACY               CONST64U(0x8000000000000000) /* Uplink is compatible with vmklinux drivers */

#endif // _VMNET_DEF_H_
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/vmxnet3_drv.c��������������������������������������������������������������������������0000644�0000000�0000000�00000406440�13432725331�014575� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "driver-config.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0)
#error "vmxnet3 driver is not supported on kernels earlier than 2.6"
#endif

#include "vm_device_version.h"

#include "vmxnet3_int.h"
#include "vmxnet3_shm.h"

char vmxnet3_driver_name[] = "vmxnet3";
#define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"

static const struct pci_device_id vmxnet3_pciid_table[] = {
	{PCI_DEVICE(PCI_VENDOR_ID_VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
	{0}
};

MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
static int disable_lro;
#endif

/* drop checker settings */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
/* if we can get udp stats, we can grow the ring larger safetly */
#define VMXNET3_RX_RING_MAX_GROWN_SIZE (VMXNET3_DEF_RX_RING_SIZE*4)
#else
/* otherwise, we should use a smaller cap on the ring growth */
#define VMXNET3_RX_RING_MAX_GROWN_SIZE (VMXNET3_DEF_RX_RING_SIZE*2)
#endif

/* Number of drops per interval which are ignored */
static u32 drop_check_noise;
/* Threshold for growing the ring */
static u32 drop_check_grow_threshold = 60;
/* Interval between packet loss checks (in seconds) */
static u32 drop_check_interval = 1;
/* Threshold for shrinking the ring */
static u32 drop_check_shrink_threshold = 60*60*24;
/* Delay before starting drop checker after ring resize or ifup */
static u32 drop_check_delay = 60;
/*
 * Percentage of Out of Buffer drops vs UDP drops which we must
 * maintain to grow the ring
 */
static u8 drop_check_min_oob_percent = 75;

static int devices_found;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 2)
static unsigned int num_enable_shm;
#endif
#define VMXNET3_SHM_MAX_DEVICES 10
static int enable_shm[VMXNET3_SHM_MAX_DEVICES + 1] =
{ [0 ... VMXNET3_SHM_MAX_DEVICES] = -1 };
static char *shm_disclaimer = NULL;
static int correct_shm_disclaimer;
static int shm_pool_size = SHM_DEFAULT_DATA_SIZE;
#ifdef VMXNET3_RSS
static unsigned int num_rss_entries = 0;

static int rss_ind_table[VMXNET3_SHM_MAX_DEVICES * VMXNET3_RSS_IND_TABLE_SIZE + 1] =
{ [0 ... VMXNET3_SHM_MAX_DEVICES * VMXNET3_RSS_IND_TABLE_SIZE] = -1 };
#endif
#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 25) || \
    defined(CONFIG_NETDEVICES_MULTIQUEUE)
static int num_tqs[VMXNET3_SHM_MAX_DEVICES + 1] =
{ [0 ... VMXNET3_SHM_MAX_DEVICES] = 0 };
#endif
#ifdef VMXNET3_RSS
static int num_rqs[VMXNET3_SHM_MAX_DEVICES + 1] =
{ [0 ... VMXNET3_SHM_MAX_DEVICES] = 0 };
#endif
static int share_tx_intr[VMXNET3_SHM_MAX_DEVICES + 1] =
{ [0 ... VMXNET3_SHM_MAX_DEVICES] = 0 };
static int buddy_intr[VMXNET3_SHM_MAX_DEVICES + 1] =
{ [0 ... VMXNET3_SHM_MAX_DEVICES] = 1 };
#define VMXNET3_SHM_DISCLAIMER "IReallyWantThisModeIAmAVMwarePartner"

static void
vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac);

/*
 *    Enable/Disable the given intr
 */

static inline void
vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
{
	VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
}

static inline void
vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
{
	VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
}

/*
 *    Enable/Disable all intrs used by the device
 */

static void
vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
{
	int i;

for (i = 0; i < adapter->intr.num_intrs; i++)
		vmxnet3_enable_intr(adapter, i);
        adapter->shared->devRead.intrConf.intrCtrl &= ~VMXNET3_IC_DISABLE_ALL;
}

static void
vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
{
	int i;

adapter->shared->devRead.intrConf.intrCtrl |= VMXNET3_IC_DISABLE_ALL;
	for (i = 0; i < adapter->intr.num_intrs; i++)
		vmxnet3_disable_intr(adapter, i);
}

static INLINE void
vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
{
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
}

static inline Bool
vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
{
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
	return netif_queue_stopped(adapter->netdev);
#else
	return __netif_subqueue_stopped(adapter->netdev, tq->qid);
#endif
}

/*
 *
 * Request the stack to start/stop/wake the tq. This only deals with the OS
 * side, it does NOT handle the device side
 */
static inline void
vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter  *adapter)
{
	tq->stopped = FALSE;
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
	netif_start_queue(adapter->netdev);
#else
	netif_start_subqueue(adapter->netdev, tq->qid);
#endif
}

static inline void
vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter  *adapter)
{
	tq->stopped = FALSE;
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
	netif_wake_queue(adapter->netdev);
#else
	netif_wake_subqueue(adapter->netdev, tq->qid);
#endif
}

static inline void
vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter  *adapter)
{
	tq->stopped = TRUE;
	tq->num_stop++;
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
	netif_stop_queue(adapter->netdev);
#else
	netif_stop_subqueue(adapter->netdev, tq->qid);
#endif
}

/*
 * This may start or stop the tx queue.
 */

static void
vmxnet3_check_link(struct vmxnet3_adapter *adapter, Bool affectTxQueue)
{
	unsigned long flags;
	u32 ret;
	int i;

spin_lock_irqsave(&adapter->cmd_lock, flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
	ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
	adapter->link_speed = ret >> 16;
	if (ret & 1) { /* Link is up. */
		printk(KERN_INFO "%s: NIC Link is Up %d Mbps\n",
		       adapter->netdev->name, adapter->link_speed);
		if (!netif_carrier_ok(adapter->netdev))
			netif_carrier_on(adapter->netdev);

if (affectTxQueue) {
			for (i = 0; i < adapter->num_tx_queues; i++)
				vmxnet3_tq_start(&adapter->tx_queue[i],
						 adapter);
		}
	} else {
		printk(KERN_INFO "%s: NIC Link is Down\n",
		       adapter->netdev->name);
		if (netif_carrier_ok(adapter->netdev))
			netif_carrier_off(adapter->netdev);

if (affectTxQueue) {
 			for (i = 0; i < adapter->num_tx_queues; i++)
				vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
		}
	}
}

/*
 * process events indicated in ECR
 */

static void
vmxnet3_process_events(struct vmxnet3_adapter *adapter)
{
	int i;
	unsigned long flags;
	u32 events = le32_to_cpu(adapter->shared->ecr);
	if (!events)
		return;

vmxnet3_ack_events(adapter, events);

/* Check if link state has changed */
	if (events & VMXNET3_ECR_LINK)
		vmxnet3_check_link(adapter, TRUE);

/* Check if there is an error on xmit/recv queues */
	if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
		spin_lock_irqsave(&adapter->cmd_lock, flags);
		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
				       VMXNET3_CMD_GET_QUEUE_STATUS);
		spin_unlock_irqrestore(&adapter->cmd_lock, flags);

for (i = 0; i < adapter->num_tx_queues; i++)
			if (adapter->tqd_start[i].status.stopped)
				printk(KERN_INFO "%s: tq[%d] error 0x%x\n",
				       adapter->netdev->name, i,
				       le32_to_cpu(adapter->tqd_start[i].status.error));
		for (i = 0; i < adapter->num_rx_queues; i++)
			if (adapter->rqd_start[i].status.stopped)
				printk(KERN_INFO "%s: rq[%d] error 0x%x\n",
				       adapter->netdev->name, i,
				       adapter->rqd_start[i].status.error);

compat_schedule_work(&adapter->reset_work);
	}
}

#ifdef __BIG_ENDIAN_BITFIELD
/*
 * The device expects the bitfields in shared structures to be written in
 * little endian. When CPU is big endian, the following routines are used to
 * correctly read and write into ABI.
 * The general technique used here is : double word bitfields are defined in
 * opposite order for big endian architecture. Then before reading them in
 * driver the complete double word is translated using le32_to_cpu. Similarly
 * After the driver writes into bitfields, cpu_to_le32 is used to translate the
 * double words into required format.
 * In order to avoid touching bits in shared structure more than once, temporary
 * descriptors are used. These are passed as srcDesc to following functions.
 */
static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
				struct Vmxnet3_RxDesc *dstDesc)
{
	u32 *src = (u32 *)srcDesc + 2;
	u32 *dst = (u32 *)dstDesc + 2;
	dstDesc->addr = le64_to_cpu(srcDesc->addr);
	*dst = le32_to_cpu(*src);
	dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
}

static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
			       struct Vmxnet3_TxDesc *dstDesc)
{
	int i;
	u32 *src = (u32 *)(srcDesc + 1);
	u32 *dst = (u32 *)(dstDesc + 1);

/* Working backwards so that the gen bit is set at the end. */
	for (i = 2; i > 0; i--) {
		src--;
		dst--;
		*dst = cpu_to_le32(*src);
	}
}

static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
				struct Vmxnet3_RxCompDesc *dstDesc)
{
	int i = 0;
	u32 *src = (u32 *)srcDesc;
	u32 *dst = (u32 *)dstDesc;
	for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
		*dst = le32_to_cpu(*src);
		src++;
		dst++;
	}
}

/* Used to read bitfield values from double words. */
static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
{
	u32 temp = le32_to_cpu(*bitfield);
	u32 mask = ((1 << size) - 1) << pos;
	temp &= mask;
	temp >>= pos;
	return temp;
}

#endif  /* __BIG_ENDIAN_BITFIELD */

#ifdef __BIG_ENDIAN_BITFIELD

#   define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
			txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
			VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
#   define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
			txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
			VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
#   define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
			VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
			VMXNET3_TCD_GEN_SIZE)
#   define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
			VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
#   define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
			(dstrcd) = (tmp); \
			vmxnet3_RxCompToCPU((rcd), (tmp)); \
		} while (0)
#   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
			(dstrxd) = (tmp); \
			vmxnet3_RxDescToCPU((rxd), (tmp)); \
		} while (0)

#else

#   define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
#   define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
#   define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
#   define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
#   define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
#   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)

#endif /* __BIG_ENDIAN_BITFIELD  */

static void
vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
		     struct pci_dev *pdev)
{
	if (tbi->map_type == VMXNET3_MAP_SINGLE)
		pci_unmap_single(pdev, tbi->dma_addr, tbi->len,
				 PCI_DMA_TODEVICE);
	else if (tbi->map_type == VMXNET3_MAP_PAGE)
		pci_unmap_page(pdev, tbi->dma_addr, tbi->len,
			       PCI_DMA_TODEVICE);
	else
		BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);

tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
}

/*
 *    Returns # of tx descs that this pkt used
 *
 * Side-effects:
 *    1. mappings are freed
 *    2. buf_info[] are updated
 *    3. tx_ring.{avail, next2comp} are updated.
 */

static int
vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
		  struct pci_dev *pdev,	struct vmxnet3_adapter *adapter)
{
	struct sk_buff *skb;
	int entries = 0;

/* no out of order completion */
	BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
	BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);

dev_dbg(&adapter->pdev->dev, "tx complete [%u %u]\n",
		tq->tx_ring.next2comp, eop_idx);
	skb = tq->buf_info[eop_idx].skb;
	BUG_ON(skb == NULL);
	tq->buf_info[eop_idx].skb = NULL;

VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);

while (tq->tx_ring.next2comp != eop_idx) {
		vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
				     pdev);

/* update next2comp w/o tx_lock. Since we are marking more,
		 * instead of less, tx ring entries avail, the worst case is
		 * that the tx routine incorrectly re-queues a pkt due to
		 * insufficient tx ring entries.
		 */
		vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
		entries++;
	}

vmxnet3_dev_kfree_skb_any(adapter, skb);
	return entries;
}

static int
vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
		       struct vmxnet3_adapter *adapter)
{
	int completed = 0;
	union Vmxnet3_GenericDesc *gdesc;

gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
	while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
		completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
					       &gdesc->tcd), tq, adapter->pdev,
					       adapter);

vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
		gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
	}

if (completed) {
		spin_lock(&tq->tx_lock);
		if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
			     vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
			     VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
			     netif_carrier_ok(adapter->netdev))) {
			vmxnet3_tq_wake(tq, adapter);
		}
		spin_unlock(&tq->tx_lock);
	}
	return completed;
}

static void
vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
		   struct vmxnet3_adapter *adapter)
{
	while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
		struct vmxnet3_tx_buf_info *tbi;

tbi = tq->buf_info + tq->tx_ring.next2comp;

vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
		if (tbi->skb) {
			vmxnet3_dev_kfree_skb_any(adapter, tbi->skb);
			tbi->skb = NULL;
		}
		vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
	}

/* sanity check */
#ifdef VMX86_DEBUG
	{
		/* verify all buffers are indeed unmapped and freed */
		int i;
		for (i = 0; i < tq->tx_ring.size; i++) {
			BUG_ON(tq->buf_info[i].skb != NULL ||
			       tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
		}
	}
#endif

tq->tx_ring.gen = VMXNET3_INIT_GEN;
	tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;

tq->comp_ring.gen = VMXNET3_INIT_GEN;
	tq->comp_ring.next2proc = 0;
}

static void
vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
{
	int i;

for (i = 0; i < adapter->num_tx_queues; i++) {
		vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
	}
}

/*
 *	free rings and buf_info for the tx queue. There must be no pending pkt
 *	in the tx ring. the .base fields of all rings and buf_info will be
 *	set to NULL
 */

static void
vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
		   struct vmxnet3_adapter *adapter)
{
	if (tq->tx_ring.base) {
		pci_free_consistent(adapter->pdev, tq->tx_ring.size *
				    sizeof(struct Vmxnet3_TxDesc),
				    tq->tx_ring.base, tq->tx_ring.basePA);
		tq->tx_ring.base = NULL;
	}
	if (tq->data_ring.base) {
		pci_free_consistent(adapter->pdev, tq->data_ring.size *
				    sizeof(struct Vmxnet3_TxDataDesc),
				    tq->data_ring.base, tq->data_ring.basePA);
		tq->data_ring.base = NULL;
	}
	if (tq->comp_ring.base) {
		pci_free_consistent(adapter->pdev, tq->comp_ring.size *
				    sizeof(struct Vmxnet3_TxCompDesc),
				    tq->comp_ring.base, tq->comp_ring.basePA);
		tq->comp_ring.base = NULL;
	}
	if (tq->buf_info) {
		pci_free_consistent(adapter->pdev,
				    tq->tx_ring.size * sizeof(tq->buf_info[0]),
				    tq->buf_info, tq->buf_info_pa);
		tq->buf_info = NULL;
	}
}

/* Destroy all tx queues */
void
vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
{
	int i;

for (i = 0; i < adapter->num_tx_queues; i++) {
		vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
	}
}

/*
 *    reset all internal states and rings for a tx queue
 * Side-effects:
 *    1. contents of the rings are reset to 0
 *    2. indices and gen of rings are reset
 *    3. bookkeeping data is reset
 */
static void
vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
		struct vmxnet3_adapter *adapter)
{
	int i;

/* reset the tx ring contents to 0 and reset the tx ring states */
	memset(tq->tx_ring.base, 0,
	       tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc));
	tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
	tq->tx_ring.gen = VMXNET3_INIT_GEN;

memset(tq->data_ring.base, 0,
	       tq->data_ring.size * sizeof(struct Vmxnet3_TxDataDesc));

/* reset the tx comp ring contents to 0 and reset comp ring states */
	memset(tq->comp_ring.base, 0,
	       tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc));
	tq->comp_ring.next2proc = 0;
	tq->comp_ring.gen = VMXNET3_INIT_GEN;

/* reset the bookkeeping data */
	memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
	for (i = 0; i < tq->tx_ring.size; i++)
		tq->buf_info[i].map_type = VMXNET3_MAP_NONE;

/* stats are not reset */
}

/* Init all tx queues */
static void
vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
{
	int i;

for (i = 0; i < adapter->num_tx_queues; i++) {
		vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
	}
}

/*
 * allocate and initialize rings for the tx queue, also allocate and
 * initialize buf_info. Returns 0 on success, negative errno on failure.
 */
static int
vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
		  struct vmxnet3_adapter *adapter)
{
	size_t sz;

BUG_ON(tq->tx_ring.size <= 0 ||
	       tq->data_ring.size != tq->tx_ring.size);
	BUG_ON((tq->tx_ring.size & VMXNET3_RING_SIZE_MASK) != 0);
	BUG_ON(tq->tx_ring.base || tq->data_ring.base || tq->comp_ring.base ||
	       tq->buf_info);

tq->tx_ring.base = pci_alloc_consistent(adapter->pdev,
			   tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
			   &tq->tx_ring.basePA);
	if (!tq->tx_ring.base) {
		printk(KERN_ERR "%s: failed to allocate tx ring\n",
		       adapter->netdev->name);
		goto err;
	}

tq->data_ring.base = pci_alloc_consistent(adapter->pdev,
			     tq->data_ring.size *
			     sizeof(struct Vmxnet3_TxDataDesc),
			     &tq->data_ring.basePA);
	if (!tq->data_ring.base) {
		printk(KERN_ERR "%s: failed to allocate data ring\n",
		       adapter->netdev->name);
		goto err;
	}

tq->comp_ring.base = pci_alloc_consistent(adapter->pdev,
			     tq->comp_ring.size *
			     sizeof(struct Vmxnet3_TxCompDesc),
			     &tq->comp_ring.basePA);
	if (!tq->comp_ring.base) {
		printk(KERN_ERR "%s: failed to allocate tx comp ring\n",
		       adapter->netdev->name);
		goto err;
	}

sz = tq->tx_ring.size * sizeof(tq->buf_info[0]);
	tq->buf_info = pci_alloc_consistent(adapter->pdev, sz,
					    &tq->buf_info_pa);
	if (!tq->buf_info) {
		printk(KERN_ERR "%s: failed to allocate tx bufinfo\n",
		       adapter->netdev->name);
		goto err;
	}
	memset(tq->buf_info, 0, sz);

return 0;

err:
	vmxnet3_tq_destroy(tq, adapter);
	return -ENOMEM;
}

/*
 *    starting from ring->next2fill, allocate rx buffers for the given ring
 *    of the rx queue and update the rx desc. stop after @num_to_alloc buffers
 *    are allocated or allocation fails. Returns # of buffers allocated
 *
 * Side-effects:
 *    1. rx descs are updated
 *    2. ring->{gen, next2fill} are updated
 *    3. uncommitted[ring_idx] is incremented
 */

static int
vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
			int num_to_alloc, struct vmxnet3_adapter *adapter)
{
	int num_allocated = 0;
	struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
	struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
	u32 val;

while (num_allocated <= num_to_alloc) {
		struct vmxnet3_rx_buf_info *rbi;
		union Vmxnet3_GenericDesc *gd;

rbi = rbi_base + ring->next2fill;
		gd = ring->base + ring->next2fill;

if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
			if (rbi->skb == NULL) {
				rbi->skb = vmxnet3_dev_alloc_skb(adapter,
						rbi->len + COMPAT_NET_IP_ALIGN);
				if (unlikely(rbi->skb == NULL)) {
					rq->stats.rx_buf_alloc_failure++;
					/* starvation prevention */
					break;
				} else if (!adapter->is_shm)
					skb_reserve(rbi->skb, NET_IP_ALIGN);
				rbi->skb->dev = adapter->netdev;
				rbi->dma_addr = vmxnet3_map_single(
						adapter, rbi->skb, 0,
						rbi->len, PCI_DMA_FROMDEVICE);
			} else {
				/* rx buffer skipped by the device */
			}
			val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
		} else {
			BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
			       rbi->len  != PAGE_SIZE);

if (rbi->page == NULL) {
				rbi->page = vmxnet3_alloc_page(adapter);
				if (unlikely(rbi->page == NULL)) {
					rq->stats.rx_buf_alloc_failure++;
					break;
				}
				rbi->dma_addr = vmxnet3_map_page(adapter,
						rbi->page, 0, PAGE_SIZE,
						PCI_DMA_FROMDEVICE);
			} else {
				/* rx buffers skipped by the device */
			}
			val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
		}

BUG_ON(rbi->dma_addr == 0);
		gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
		gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
					   | val | rbi->len);

/*Fill the last buffer but do not mark it ready */
		if (num_allocated == num_to_alloc)
			break;

gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);

num_allocated++;
		vmxnet3_cmd_ring_adv_next2fill(ring);
	}
	rq->uncommitted[ring_idx] += num_allocated;

dev_dbg(&adapter->pdev->dev, "alloc_rx_buf: %d allocated, next2fill "
		"%u, next2comp %u, uncommitted %u\n", num_allocated,
		ring->next2fill, ring->next2comp, rq->uncommitted[ring_idx]);

/* so that the device can distinguish a full ring and an empty ring */
	BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);

return num_allocated;
}

/*
 * It assumes the skb still has space to accommodate the frag. It only
 * increments skb->data_len
 */

static void
vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
		    struct vmxnet3_rx_buf_info *rbi)
{
	struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
		skb_shinfo(skb)->nr_frags;

BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);

frag->page_offset = 0;
#if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
	frag->page = rbi->page;
	frag->size = rcd->len;
	skb->data_len += frag->size;
#else
	__skb_frag_set_page(frag, rbi->page);
	skb_frag_size_set(frag, rcd->len);
	skb->data_len += rcd->len;
#endif
	skb->truesize += PAGE_SIZE;
	skb_shinfo(skb)->nr_frags++;
}

/*
 * Map the tx buffer and set up ONLY TXD.{addr, len, gen} based on the mapping.
 * It sets the other fields of the descriptors to 0.
 * Side Effects :
 *    1. the corresponding buf_info entries are upated,
 *    2. ring indices are advanced
 */

static void
vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
		struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
		struct vmxnet3_adapter *adapter)
{
	u32 dw2, len;
	unsigned long buf_offset;
	int i;
	union Vmxnet3_GenericDesc *gdesc;
	struct vmxnet3_tx_buf_info *tbi = NULL;

BUG_ON(ctx->copy_size > vmxnet3_skb_headlen(adapter, skb));

/* use the previous gen bit for the SOP desc */
	dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;

ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
	gdesc = ctx->sop_txd; /* both loops below can be skipped */

/* no need to map the buffer if headers are copied */
	if (ctx->copy_size) {
		BUG_ON(VMXNET3_TXDESC_GET_GEN(&(ctx->sop_txd->txd)) ==
		       tq->tx_ring.gen);

ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
					tq->tx_ring.next2fill *
					sizeof(struct Vmxnet3_TxDataDesc));
		ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
		ctx->sop_txd->dword[3] = 0;

tbi = tq->buf_info + tq->tx_ring.next2fill;
		tbi->map_type = VMXNET3_MAP_NONE;

dev_dbg(&adapter->pdev->dev, "txd[%u]: 0x%llu 0x%x 0x%x\n",
			tq->tx_ring.next2fill,
			le64_to_cpu(ctx->sop_txd->txd.addr),
			ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
		vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);

/* use the right gen for non-SOP desc */
		dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
	}

/* linear part can use multiple tx desc if it's big */
	len = vmxnet3_skb_headlen(adapter, skb) - ctx->copy_size;
	buf_offset = ctx->copy_size;
	while (len) {
		u32 buf_size;

if (len < VMXNET3_MAX_TX_BUF_SIZE) {
			buf_size = len;
			dw2 |= len;
		} else {
			buf_size = VMXNET3_MAX_TX_BUF_SIZE;
			/* spec says that for TxDesc.len, 0 == 2^14 */
		}

tbi = tq->buf_info + tq->tx_ring.next2fill;
		tbi->map_type = VMXNET3_MAP_SINGLE;
		tbi->dma_addr = vmxnet3_map_single(adapter, skb, buf_offset,
				buf_size, PCI_DMA_TODEVICE);

tbi->len = buf_size;

gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
		BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);

gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
		gdesc->dword[2] = cpu_to_le32(dw2);
		gdesc->dword[3] = 0;

dev_dbg(&adapter->pdev->dev,
			"txd[%u]: 0x%llu 0x%x 0x%x\n",
			tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
			le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
		vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
		dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;

len -= buf_size;
		buf_offset += buf_size;

if (adapter->is_shm) {
		/*
		 * The linear region of the skb is never larger than a page, so
		 * it always fits into one descriptor.
		 */
			BUG_ON(len != 0);
		}
	}

for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
		const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
		u32 buf_size;

buf_offset = frag->page_offset;
#if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
		len = frag->size;
#else
		len = skb_frag_size(frag);
#endif
		while (len) {
			tbi = tq->buf_info + tq->tx_ring.next2fill;
			if (len < VMXNET3_MAX_TX_BUF_SIZE) {
				buf_size = len;
				dw2 |= len;
			} else {
				buf_size = VMXNET3_MAX_TX_BUF_SIZE;
				/* spec says that for TxDesc.len, 0 == 2^14 */
			}
			tbi->map_type = VMXNET3_MAP_PAGE;
#if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
			tbi->dma_addr = vmxnet3_map_page(adapter, frag->page,
							 buf_offset, buf_size,
							 DMA_TO_DEVICE);
#else
			tbi->dma_addr = vmxnet3_map_page(adapter, frag->page.p,
							 buf_offset, buf_size,
							 DMA_TO_DEVICE);
#endif
			tbi->len = buf_size;
			gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
			BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);

gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
			gdesc->dword[2] = cpu_to_le32(dw2);
			gdesc->dword[3] = 0;

vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
			dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
			dev_dbg(&adapter->pdev->dev,
				"txd[%u]: 0x%llu %u %u\n",
				tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
				le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
			len -= buf_size;
			buf_offset += buf_size;
		}
	}

ctx->eop_txd = gdesc;

/* set the last buf_info for the pkt */
	tbi->skb = skb;
	tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
}

/*
 *    parse and copy relevant protocol headers:
 *     For a tso pkt, relevant headers are L2/3/4 including options
 *     For a pkt requesting csum offloading, they are L2/3 and may include L4
 *     if it's a TCP/UDP pkt
 *
 *    The implementation works only when h/w vlan insertion is used, see PR
 *    171928
 *
 * Result:
 *    -1:  error happens during parsing
 *     0:  protocol headers parsed, but too big to be copied
 *     1:  protocol headers parsed and copied
 *
 * Side-effects:
 *    1. related *ctx fields are updated.
 *    2. ctx->copy_size is # of bytes copied
 *    3. the portion copied is guaranteed to be in the linear part
 *
 */
static int
vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
			   struct vmxnet3_tx_ctx *ctx,
			   struct vmxnet3_adapter *adapter)
{
	struct Vmxnet3_TxDataDesc *tdd;
	unsigned char protocol = 0;

if (ctx->mss) {	/* TSO */
		ctx->eth_ip_hdr_size = compat_skb_transport_offset(skb);
		ctx->l4_hdr_size = compat_skb_tcp_header(skb)->doff * 4;
		ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;
	} else {
		if (skb->ip_summed == VM_TX_CHECKSUM_PARTIAL) {
			ctx->eth_ip_hdr_size = compat_skb_transport_offset(skb);

if (ctx->ipv4) {
				struct iphdr *iph = (struct iphdr *)
						    compat_skb_ip_header(skb);
				protocol = iph->protocol;
			} else if (ctx->ipv6) {
				struct ipv6hdr *ipv6h =
					(struct ipv6hdr *)
					compat_skb_ipv6_header(skb);
				protocol = ipv6h->nexthdr;
			}
			if (protocol == IPPROTO_TCP)
				ctx->l4_hdr_size =
					compat_skb_tcp_header(skb)->doff * 4;
			else if (protocol == IPPROTO_UDP)
				ctx->l4_hdr_size =
					sizeof(struct udphdr);
			else
				ctx->l4_hdr_size = 0;

/* make sure that copy size is not exceeding pkt len */
			ctx->copy_size = min((ctx->eth_ip_hdr_size +
                                              ctx->l4_hdr_size), skb->len);
		} else {
			ctx->eth_ip_hdr_size = 0;
			ctx->l4_hdr_size = 0;
			/* copy as much as allowed */
			ctx->copy_size = min((unsigned int)
                                             VMXNET3_HDR_COPY_SIZE,
					     vmxnet3_skb_headlen(adapter, skb));
		}

if(!adapter->is_shm) {
			if (skb->len <= VMXNET3_HDR_COPY_SIZE) {
				ctx->copy_size = skb->len;
			}
			/* make sure headers are accessible directly */
			if (unlikely(!compat_pskb_may_pull(skb,
							   ctx->copy_size)))
				goto err;

}
	}

if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
		tq->stats.oversized_hdr++;
		ctx->copy_size = 0;
		return 0;
	}

tdd = tq->data_ring.base + tq->tx_ring.next2fill;
	BUG_ON(ctx->copy_size > vmxnet3_skb_headlen(adapter, skb));

if (!adapter->is_shm) {
		memcpy(tdd->data, skb->data, ctx->copy_size);
	} else {
		void *virt = compat_kmap_atomic(VMXNET3_SHM_IDX2PAGE(adapter->shm,
					        VMXNET3_SHM_SKB_GETIDX(skb)));
		memcpy(tdd->data, virt, ctx->copy_size);
		compat_kunmap_atomic(virt);
	}

dev_dbg(&adapter->pdev->dev, "copy %u bytes to dataRing[%u]\n",
		ctx->copy_size, tq->tx_ring.next2fill);
	return 1;

err:
	return -1;
}

/*
 *    Fix pkt headers for tso. ip hdr and tcp hdr are changed
 */

static void
vmxnet3_prepare_tso(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx)
{
	struct tcphdr *tcph = compat_skb_tcp_header(skb);
	if (ctx->ipv4) {
		struct iphdr *iph = compat_skb_ip_header(skb);
		iph->check = 0;
		tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
						 IPPROTO_TCP, 0);
#ifdef NETIF_F_TSO6
	} else if (ctx->ipv6) {
		struct ipv6hdr *iph = (struct ipv6hdr *)
				      compat_skb_network_header(skb);
		tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
					       IPPROTO_TCP, 0);
#endif
	}
}

inline void vmxnet3_le32_add_cpu(uint32 *addTo, uint32 addThis)
{
	*addTo = cpu_to_le32(le32_to_cpu(*addTo) + addThis);
}

static int txd_estimate(const struct sk_buff *skb)
{
	int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
	int i;

for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
		const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
#if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
		count += VMXNET3_TXD_NEEDED(frag->size);
#else
		count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
#endif
	}
	return count;
}

/*
 *    transmit a pkt thru a given tq
 *
 * Result:
 *    COMPAT_NETDEV_TX_OK:      descriptors are setup successfully
 *    COMPAT_NETDEV_TX_OK:      error occurred, the pkt is dropped
 *    COMPAT_NETDEV_TX_BUSY:    tx ring is full, queue is stopped
 *
 * Side-effects:
 *    1. tx ring may be changed
 *    2. tq stats may be updated accordingly
 *    3. shared->txNumDeferred may be updated
 */

int
vmxnet3_tq_xmit(struct sk_buff *skb,
		struct vmxnet3_tx_queue *tq,
		struct vmxnet3_adapter *adapter,
		struct net_device *netdev)
{
	int ret;
	u32 count;
	int num_pkts;
	int local_tx_num_deferred;
	unsigned long flags;
	struct vmxnet3_tx_ctx ctx;
	union Vmxnet3_GenericDesc *gdesc;
#ifdef __BIG_ENDIAN_BITFIELD
	/* Use temporary descriptor to avoid touching bits multiple times */
	union Vmxnet3_GenericDesc tempTxDesc;
#endif

/* conservatively estimate # of descriptors to use */
	count = txd_estimate(skb);
	ctx.ipv4 = (compat_vlan_get_protocol(skb) == __constant_ntohs(ETH_P_IP));
	ctx.ipv6 = (compat_vlan_get_protocol(skb) == __constant_ntohs(ETH_P_IPV6));

ctx.mss = compat_skb_mss(skb);
	if (ctx.mss) {
		if (compat_skb_header_cloned(skb)) {
			if (unlikely(pskb_expand_head(skb, 0, 0,
						      GFP_ATOMIC) != 0)) {
				tq->stats.drop_tso++;
				goto drop_pkt;
			}
			tq->stats.copy_skb_header++;
		}
		vmxnet3_prepare_tso(skb, &ctx);
	} else {
		if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
			if (unlikely(adapter->is_shm)) {
				BUG_ON(count > VMXNET3_MAX_TXD_PER_PKT_SHM);
				if (count > VMXNET3_MAX_TXD_PER_PKT_SHM) {
					tq->stats.drop_too_many_frags++;
					goto drop_pkt;
				}
			} else {
				/* non-tso pkts must not use more than
				 * VMXNET3_MAX_TXD_PER_PKT entries
				 */
				if (compat_skb_linearize(skb) != 0) {
					tq->stats.drop_too_many_frags++;
					goto drop_pkt;
				}
				tq->stats.linearized++;

/* recalculate the # of descriptors to use */
				count = VMXNET3_TXD_NEEDED(vmxnet3_skb_headlen(
							adapter, skb)) + 1;
			}
		}
	}

spin_lock_irqsave(&tq->tx_lock, flags);

if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
		tq->stats.tx_ring_full++;
		dev_dbg(&adapter->pdev->dev, "tx queue stopped on %s, next2comp"
			" %u next2fill %u\n", adapter->netdev->name,
			tq->tx_ring.next2comp, tq->tx_ring.next2fill);

vmxnet3_tq_stop(tq, adapter);
		spin_unlock_irqrestore(&tq->tx_lock, flags);
		return COMPAT_NETDEV_TX_BUSY;
	}

ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
	if (ret >= 0) {
		BUG_ON(ret <= 0 && ctx.copy_size != 0);
		/* hdrs parsed, check against other limits */
		if (ctx.mss) {
			if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >
				     VMXNET3_MAX_TX_BUF_SIZE)) {
				goto hdr_too_big;
			}
		} else {
			if (skb->ip_summed == VM_TX_CHECKSUM_PARTIAL) {
				if (unlikely(ctx.eth_ip_hdr_size +
					     compat_skb_csum_offset(skb) >
					     VMXNET3_MAX_CSUM_OFFSET)) {
					goto hdr_too_big;
				}
			}
		}
	} else {
		tq->stats.drop_hdr_inspect_err++;
		goto unlock_drop_pkt;
	}

/* fill tx descs related to addr & len */
	vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);

/* setup the EOP desc */
	ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);

/* setup the SOP desc */
#ifdef __BIG_ENDIAN_BITFIELD
	gdesc = &tempTxDesc;
	gdesc->dword[2] = ctx.sop_txd->dword[2];
	gdesc->dword[3] = ctx.sop_txd->dword[3];
#else
	gdesc = ctx.sop_txd;
#endif
	local_tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred);
	if (ctx.mss) {
		gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
		gdesc->txd.om = VMXNET3_OM_TSO;
		gdesc->txd.msscof = ctx.mss;
		num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss;
	} else {
		if (skb->ip_summed == VM_TX_CHECKSUM_PARTIAL) {
			gdesc->txd.hlen = ctx.eth_ip_hdr_size;
			gdesc->txd.om = VMXNET3_OM_CSUM;
			gdesc->txd.msscof = ctx.eth_ip_hdr_size +
					    compat_skb_csum_offset(skb);
		} else {
			gdesc->txd.om = 0;
			gdesc->txd.msscof = 0;
		}
		num_pkts = 1;
	}
	vmxnet3_le32_add_cpu(&tq->shared->txNumDeferred, num_pkts);
	local_tx_num_deferred += num_pkts;

if (vlan_tx_tag_present(skb)) {
		gdesc->txd.ti = 1;
		gdesc->txd.tci = vlan_tx_tag_get(skb);
	}

/* finally flips the GEN bit of the SOP desc. */
	gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
						  VMXNET3_TXD_GEN);
#ifdef __BIG_ENDIAN_BITFIELD
	/* Finished updating in bitfields of Tx Desc, so write them in original
	 * place.
	 */
	vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
			   (struct Vmxnet3_TxDesc *)ctx.sop_txd);
	gdesc = ctx.sop_txd;
#endif
	dev_dbg(&adapter->pdev->dev,
		"txd[%u]: SOP 0x%llu 0x%x 0x%x\n",
		(u32)(ctx.sop_txd -
		tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
		le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));

spin_unlock_irqrestore(&tq->tx_lock, flags);

if (local_tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) {
		tq->shared->txNumDeferred = 0;
		VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_TXPROD +
				       tq->qid * 8), tq->tx_ring.next2fill);
	}
	netdev->trans_start = jiffies;

return COMPAT_NETDEV_TX_OK;

hdr_too_big:
	tq->stats.drop_oversized_hdr++;
unlock_drop_pkt:
	spin_unlock_irqrestore(&tq->tx_lock, flags);
drop_pkt:
	tq->stats.drop_total++;
	vmxnet3_dev_kfree_skb(adapter, skb);
	return COMPAT_NETDEV_TX_OK;
}

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
static int
#else
static netdev_tx_t
#endif
vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);

if (adapter->is_shm) {
		return vmxnet3_shm_start_tx(skb, netdev);
	} else {
#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 25) || defined(CONFIG_NETDEVICES_MULTIQUEUE)
		BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
		return vmxnet3_tq_xmit(skb,
				       &adapter->tx_queue[skb->queue_mapping],
				       adapter, netdev);
#else
		return vmxnet3_tq_xmit(skb, &adapter->tx_queue[0], adapter,
				       netdev);
#endif
	}
}

/* called to process csum related bits in the EOP RCD descriptor */
static void
vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
		struct sk_buff *skb,
		union Vmxnet3_GenericDesc *gdesc)
{
	if (!gdesc->rcd.cnc && adapter->rxcsum) {
		/* typical case: TCP/UDP over IP and both csums are correct */
		/* For IPv4 check if both rcd->ipc and rcd->tuc is true */
		/* in case of IPv6 packet, just check rcd->tuc*/
		if (gdesc->rcd.v4 && (le32_to_cpu(gdesc->dword[3]) &
                                  VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) {
			skb->ip_summed = VM_CHECKSUM_UNNECESSARY;
			BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
			BUG_ON(gdesc->rcd.frg);
		} else if(gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) &
			  (1 << VMXNET3_RCD_TUC_SHIFT))) {
                        skb->ip_summed = VM_CHECKSUM_UNNECESSARY;
                        BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
                        BUG_ON(gdesc->rcd.frg);
		} else {
			if (gdesc->rcd.csum) {
				skb->csum = htons(gdesc->rcd.csum);
				skb->ip_summed = VM_RX_CHECKSUM_PARTIAL;
			} else {
				skb->ip_summed = CHECKSUM_NONE;
			}
		}
	} else {
		skb->ip_summed = CHECKSUM_NONE;
	}
}

/*
 * called when ERR bit is set for a received pkt. The desc and the associated
 * rx buffer have not been processed yet.
 *
 * Side-effects:
 *    1. up the stat counters
 *    2. free the skb if needed
 *    3. reset ctx->skb
 */

static void
vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
		 struct vmxnet3_rx_ctx *ctx,  struct vmxnet3_adapter *adapter)
{
	rq->stats.drop_err++;
	if (!rcd->fcs)
		rq->stats.drop_fcs++;

rq->stats.drop_total++;

/*
	 * We do not unmap and chain the rx buffer to the skb.
	 * We basically pretend this buffer is not used and will be recycled
	 * by vmxnet3_rq_alloc_rx_buf()
	 */

/*
	 * ctx->skb may be NULL if this is the first and the only one
	 * desc for the pkt
	 */
	if (ctx->skb)
	        vmxnet3_dev_kfree_skb_irq(adapter, ctx->skb);

ctx->skb = NULL;
}

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)

static u32
vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb,
		union Vmxnet3_GenericDesc *gdesc)
{
	u32 hlen, mapLen;
	union {
		void *ptr;
		struct ethhdr *eth;
		struct iphdr *ipv4;
		struct ipv6hdr *ipv6;
		struct tcphdr *tcp;
	} hdr;
	BUG_ON(gdesc->rcd.tcp == 0);

mapLen = vmxnet3_skb_headlen(adapter, skb);
	if (UNLIKELY(sizeof (struct iphdr) + sizeof (struct tcphdr) > mapLen)) {
		return 0;
	}

hdr.eth = eth_hdr(skb);
	if (gdesc->rcd.v4) {
		BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP));
		hdr.ptr += sizeof (struct ethhdr);
		BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP);
		hlen = hdr.ipv4->ihl << 2;
		hdr.ptr += hdr.ipv4->ihl << 2;
	} else if (gdesc->rcd.v6) {
		BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6));
		hdr.ptr += sizeof (struct ethhdr);
		/*
		 * just do estimation, since we also need to handle
		 * TSO case.
		 */
		if (hdr.ipv6->nexthdr != IPPROTO_TCP)
			return sizeof (struct ipv6hdr) + sizeof (struct tcphdr);
		hlen = sizeof (struct ipv6hdr);
		hdr.ptr += sizeof (struct ipv6hdr);
	} else {
		/* Non-IP pkt, dont estimate header length */
		return 0;
	}

if (hlen + sizeof (struct tcphdr) > mapLen) {
		return 0;
	}
	return (hlen + (hdr.tcp->doff << 2));
}
#endif

/* process the rx completion ring of the given rx queue. Quota specifies the
 * max # of rx completion entries to be processed. Returns # of descs completed.
 */
static int
#ifdef VMXNET3_NAPI
vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
		       struct vmxnet3_adapter *adapter, int quota)
#else
vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
		       struct vmxnet3_adapter *adapter)
#endif
{
	static const u32 rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
	u32 num_pkts = 0;
	Bool skip_page_frags = FALSE;
	struct Vmxnet3_RxCompDesc *rcd;
	struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
        u16 segCnt = 0, mss = 0;
#ifdef __BIG_ENDIAN_BITFIELD
	struct Vmxnet3_RxDesc rxCmdDesc;
	struct Vmxnet3_RxCompDesc rxComp;
#endif
	vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
			  &rxComp);
	while (rcd->gen == rq->comp_ring.gen) {
		struct vmxnet3_rx_buf_info *rbi;
		struct sk_buff *skb, *new_skb = NULL;
		struct page *new_page = NULL;
		int num_to_alloc;
		struct Vmxnet3_RxDesc *rxd;
		u32 idx, ring_idx;
		struct vmxnet3_cmd_ring *ring = NULL;

#ifdef VMXNET3_NAPI
		if (num_pkts >= quota) {
			/* we may stop even before we see the EOP desc of
			 * the current pkt
			 */
			break;
		}
#endif
		BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
		idx = rcd->rxdIdx;
		ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
		ring = rq->rx_ring + ring_idx;
		vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
				  &rxCmdDesc);
		rbi = rq->buf_info[ring_idx] + idx;

BUG_ON(rcd->len > rxd->len);
		BUG_ON(rxd->addr != rbi->dma_addr ||
		       rxd->len != rbi->len);

if (unlikely(rcd->eop && rcd->err)) {
			vmxnet3_rx_error(rq, rcd, ctx, adapter);
			goto rcd_done;
		}

if (rcd->sop) { /* first buf of the pkt */
			BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
			       rcd->rqID != rq->qid);

BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
			BUG_ON(ctx->skb != NULL || rbi->skb == NULL);

if (unlikely(rcd->len == 0)) {
				/* Pretend the rx buffer is skipped. */
				BUG_ON(!(rcd->sop && rcd->eop));
				dev_dbg(&adapter->pdev->dev, "rxRing[%u][%u] 0"
					" length\n", ring_idx, idx);
				goto rcd_done;
			}

skip_page_frags = FALSE;
			ctx->skb = rbi->skb;
			new_skb = vmxnet3_dev_alloc_skb(adapter, rbi->len +
							COMPAT_NET_IP_ALIGN);
			if (new_skb == NULL) {
				/* SKB allocation failed, do not handover this
				 * skb to stack. Reuse it. Drop the existing pkt
				 */
				rq->stats.rx_buf_alloc_failure++;
				ctx->skb = NULL;
				rq->stats.drop_total++;
				skip_page_frags = TRUE;
				goto rcd_done;
			}
			pci_unmap_single(adapter->pdev, rbi->dma_addr, rbi->len,
					 PCI_DMA_FROMDEVICE);

vmxnet3_skb_put(adapter, ctx->skb, rcd->len);

/* Immediate refill */
			new_skb->dev = adapter->netdev;
			if (!adapter->is_shm)
				skb_reserve(new_skb, NET_IP_ALIGN);
			rbi->skb = new_skb;
			rbi->dma_addr = vmxnet3_map_single(adapter,
					rbi->skb, 0, rbi->len,
					PCI_DMA_FROMDEVICE);
			rxd->addr = cpu_to_le64(rbi->dma_addr);
			rxd->len = rbi->len;
			if (adapter->version == 2 &&
			    rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) {
				struct Vmxnet3_RxCompDescExt *rcdLro;
				rcdLro = (struct Vmxnet3_RxCompDescExt *)rcd;
				segCnt = rcdLro->segCnt;
				BUG_ON(segCnt == 0);
				mss = rcdLro->mss;
				if (UNLIKELY(segCnt <= 1)) {
					segCnt = 0;
				}
			} else {
				segCnt = 0;
			}
		} else {
			BUG_ON(ctx->skb == NULL && !skip_page_frags);
			/* non SOP buffer must be type 1 in most cases */
			BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
			BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);

/* If an sop buffer was dropped, skip all
			 * following non-sop fragments. They will be reused.
			 */
			if (skip_page_frags)
				goto rcd_done;

if (rcd->len) {
				new_page = vmxnet3_alloc_page(adapter);
				if (unlikely(new_page == NULL)) {
				/* Replacement page frag could not be allocated.
				 * Reuse this page. Drop the pkt and free the
				 * skb which contained this page as a frag. Skip
				 * processing all the following non-sop frags.
				 */
					rq->stats.rx_buf_alloc_failure++;
					vmxnet3_dev_kfree_skb(adapter, ctx->skb);
					ctx->skb = NULL;
					skip_page_frags = TRUE;
					goto rcd_done;
				}

pci_unmap_page(adapter->pdev,
					       rbi->dma_addr, rbi->len,
					       PCI_DMA_FROMDEVICE);

vmxnet3_append_frag(ctx->skb, rcd, rbi);

/* Immediate refill */
				rbi->page = new_page;
				rbi->dma_addr = vmxnet3_map_page(adapter,
						rbi->page, 0, PAGE_SIZE,
						PCI_DMA_FROMDEVICE);
				rxd->addr = cpu_to_le64(rbi->dma_addr);
				rxd->len = rbi->len;
			}
		}

skb = ctx->skb;
		if (rcd->eop) {
			if (adapter->is_shm) {
				vmxnet3_shm_rx_skb(adapter, skb);

wake_up(&adapter->shm->rxq);
			} else {
				skb->len += skb->data_len;

vmxnet3_rx_csum(adapter, skb,
					(union Vmxnet3_GenericDesc *)rcd);
				skb->protocol = eth_type_trans(skb,
							       adapter->netdev);
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
			{
        			u32 mtu = adapter->netdev->mtu;
				if (!rcd->tcp || !adapter->lro)
					goto not_lro;

if (segCnt != 0 && mss != 0) {
					skb_shinfo(skb)->gso_type = rcd->v4 ?
						SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
					skb_shinfo(skb)->gso_size = mss;
					skb_shinfo(skb)->gso_segs = segCnt;
				} else if (segCnt != 0 || skb->len > mtu) {
					u32 hlen;
					hlen = vmxnet3_get_hdr_len(adapter, skb,
						(Vmxnet3_GenericDesc *)rcd);
					if (hlen == 0)
						goto not_lro;

skb_shinfo(skb)->gso_type =
						rcd->v4 ? SKB_GSO_TCPV4
						: SKB_GSO_TCPV6;
					if (segCnt != 0) {
						skb_shinfo(skb)->gso_segs =
									segCnt;
						skb_shinfo(skb)->gso_size =
							DIV_ROUND_UP(skb->len -
								hlen, segCnt);
					} else {
						skb_shinfo(skb)->gso_size =
							mtu - hlen;
					}
				}
			}
not_lro:
#endif
#ifdef VMXNET3_NAPI
#   if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29)
				if (unlikely(rcd->ts))
                                   if (adapter->vlan_grp) {
					vlan_hwaccel_receive_skb(skb,
						adapter->vlan_grp, rcd->tci);
                                   } else {
                                        /* If no vlan sub interface is configured
                                         * in the OS, vlan_grp structure will be
                                         * null and at this time, if there is vlan
                                         * packet on the wire, the VM will crash.
                                         * We need to avoid the crash here.
                                         */
                                        vmxnet3_dev_kfree_skb(adapter, ctx->skb);
                                        ctx->skb = NULL;
                                        goto rcd_done;
                                   }
				else
					netif_receive_skb(skb);
#   else
#      if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37)
				if (unlikely(rcd->ts)) {
                                   if (adapter->vlan_grp) {
					vlan_hwaccel_receive_skb(skb,
						adapter->vlan_grp, rcd->tci);
                                   } else {
                                        /* If no vlan sub interface is configured
                                         * in the OS, vlan_grp structure will be
                                         * null and at this time, if there is vlan
                                         * packet on the wire, the VM will crash.
                                         * We need to avoid the crash here.
                                         */
                                        vmxnet3_dev_kfree_skb(adapter, ctx->skb);
                                        ctx->skb = NULL;
                                        goto rcd_done;
                                   }
				} else {
					if (adapter->netdev->features & NETIF_F_LRO)
						netif_receive_skb(skb);
					else
						napi_gro_receive(&rq->napi, skb);
				}
#      else
				if (unlikely(rcd->ts)) {
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0)
					__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
#else
					__vlan_hwaccel_put_tag(skb, rcd->tci);
#endif
                                }
				if (adapter->netdev->features & NETIF_F_LRO)
					netif_receive_skb(skb);
				else
					napi_gro_receive(&rq->napi, skb);
#      endif
#   endif
#else
				netif_rx(skb);
#endif
			}

adapter->netdev->last_rx = jiffies;
			ctx->skb = NULL;
			num_pkts++;
		}

rcd_done:
		/* device may have skipped some rx descs */
		ring->next2comp = idx;
		num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
		ring = rq->rx_ring + ring_idx;
		while (num_to_alloc) {
			vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
					  &rxCmdDesc);
			BUG_ON(!rxd->addr);

/* Recv desc is ready to be used by the device */
			rxd->gen = ring->gen;
			vmxnet3_cmd_ring_adv_next2fill(ring);
			num_to_alloc--;
		}

/* if needed, update the register */
		if (unlikely(rq->shared->updateRxProd)) {
			VMXNET3_WRITE_BAR0_REG(adapter,
				rxprod_reg[ring_idx] + rq->qid * 8,
				ring->next2fill);
			rq->uncommitted[ring_idx] = 0;

}

vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
		vmxnet3_getRxComp(rcd,
		     &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
	}

return num_pkts;
}

/*
 * Unmap and free the rx buffers allocated to the rx queue. Other resources
 * are NOT freed. This is the counterpart of vmxnet3_rq_init()
 * Side-effects:
 *    1. indices and gen of each ring are reset to the initial value
 *    2. buf_info[] and buf_info2[] are cleared.
 */

static void
vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
		   struct vmxnet3_adapter *adapter)
{
	u32 i, ring_idx;
	struct Vmxnet3_RxDesc *rxd;

for (ring_idx = 0; ring_idx < 2; ring_idx++) {
		for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
#ifdef __BIG_ENDIAN_BITFIELD
			struct Vmxnet3_RxDesc rxDesc;
#endif
			vmxnet3_getRxDesc(rxd,
				&rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);

if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
					rq->buf_info[ring_idx][i].skb) {
				pci_unmap_single(adapter->pdev, rxd->addr,
						 rxd->len, PCI_DMA_FROMDEVICE);
				vmxnet3_dev_kfree_skb(adapter, rq->buf_info[ring_idx][i].skb);
				rq->buf_info[ring_idx][i].skb = NULL;
			} else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
					rq->buf_info[ring_idx][i].page) {
				pci_unmap_page(adapter->pdev, rxd->addr,
					       rxd->len, PCI_DMA_FROMDEVICE);
				vmxnet3_put_page(adapter,
						rq->buf_info[ring_idx][i].page);
				rq->buf_info[ring_idx][i].page = NULL;
			}
		}

rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
		rq->rx_ring[ring_idx].next2fill =
					rq->rx_ring[ring_idx].next2comp = 0;
		rq->uncommitted[ring_idx] = 0;
	}

rq->comp_ring.gen = VMXNET3_INIT_GEN;
	rq->comp_ring.next2proc = 0;
}

static void
vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
{
	int i;

for (i = 0; i < adapter->num_rx_queues; i++) {
		vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
	}
}

/*
 *    Free rings and buf_info for the rx queue. The rx buffers must have
 *    ALREADY been freed. the .base fields of all rings will be set to NULL
 */

static void
vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
		struct vmxnet3_adapter *adapter)
{
	int i;

#ifdef VMX86_DEBUG
	/* all rx buffers must have already been freed */
	{
		int j;

for (i = 0; i < 2; i++) {
			if (rq->buf_info[i]) {
				for (j = 0; j < rq->rx_ring[i].size; j++) {
					BUG_ON(rq->buf_info[i][j].page != NULL);
				}
			}
		}
	}
#endif

for (i = 0; i < 2; i++) {
		if (rq->rx_ring[i].base) {
			pci_free_consistent(adapter->pdev, rq->rx_ring[i].size
					    * sizeof(struct Vmxnet3_RxDesc),
					    rq->rx_ring[i].base,
					    rq->rx_ring[i].basePA);
			rq->rx_ring[i].base = NULL;
		}
		rq->buf_info[i] = NULL;
	}

if (rq->comp_ring.base) {
		pci_free_consistent(adapter->pdev, rq->comp_ring.size *
				    sizeof(struct Vmxnet3_RxCompDesc),
				    rq->comp_ring.base, rq->comp_ring.basePA);
		rq->comp_ring.base = NULL;
	}

if (rq->buf_info[0]) {
		size_t sz = sizeof(struct vmxnet3_rx_buf_info) *
			(rq->rx_ring[0].size + rq->rx_ring[1].size);
		pci_free_consistent(adapter->pdev, sz, rq->buf_info[0],
				    rq->buf_info_pa);
	}
}

void
vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
{
	int i;

for (i = 0; i < adapter->num_rx_queues; i++) {
		vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
	}
}

/*
 *    initialize buf_info, allocate rx buffers and fill the rx rings. On
 *    failure, the rx buffers already allocated are NOT freed
 */

static int
vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
		struct vmxnet3_adapter  *adapter)
{
	int i;

BUG_ON(adapter->rx_buf_per_pkt <= 0 ||
	       rq->rx_ring[0].size % adapter->rx_buf_per_pkt != 0);
	/* initialize buf_info */
	for (i = 0; i < rq->rx_ring[0].size; i++) {
		BUG_ON(rq->buf_info[0][i].skb != NULL);
		/* 1st buf for a pkt is skbuff */
		if (i % adapter->rx_buf_per_pkt == 0) {
			rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
			rq->buf_info[0][i].len = adapter->skb_buf_size;
		} else { /* subsequent bufs for a pkt is frag */
			rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
			rq->buf_info[0][i].len = PAGE_SIZE;
		}
	}
	for (i = 0; i < rq->rx_ring[1].size; i++) {
		BUG_ON(rq->buf_info[1][i].page != NULL);
		rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
		rq->buf_info[1][i].len = PAGE_SIZE;
	}

/* reset internal state and allocate buffers for both rings */
	for (i = 0; i < 2; i++) {
		rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
		rq->uncommitted[i] = 0;

memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
		       sizeof(struct Vmxnet3_RxDesc));
		rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
	}

/* populate initial ring */
	if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
				    adapter) == 0) {
		/* at least has 1 rx buffer for the 1st ring */
		return -ENOMEM;
	}
	vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);

/* reset the comp ring */
	rq->comp_ring.next2proc = 0;
	memset(rq->comp_ring.base, 0, rq->comp_ring.size *
	       sizeof(struct Vmxnet3_RxCompDesc));
	rq->comp_ring.gen = VMXNET3_INIT_GEN;

/* reset rxctx */
	rq->rx_ctx.skb = NULL;

/* stats are not reset */
	return 0;
}

static int
vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
{
	int i, err = 0;

for (i = 0; i < adapter->num_rx_queues; i++) {
		err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
		if (UNLIKELY(err)) {
			printk(KERN_ERR "%s: failed to initialize rx queue%i\n",
			       adapter->netdev->name, i);
			break;
		}
	}
	return err;

}

static u64
vmxnet3_get_oob_drop_count(struct vmxnet3_adapter *adapter)
{
	unsigned long flags;

spin_lock_irqsave(&adapter->cmd_lock, flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);

return adapter->rqd_start->stats.pktsRxOutOfBuf;
}

static u64
vmxnet3_get_udp_drop_count(struct vmxnet3_adapter *adapter)
{
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 33)
	struct net *net = dev_net(adapter->netdev);

return snmp_fold_field((void __percpu **)net->mib.udp_statistics,
			       UDP_MIB_RCVBUFERRORS) +
		snmp_fold_field((void __percpu **)net->mib.udplite_statistics,
   				UDP_MIB_RCVBUFERRORS);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
	struct net *net = dev_net(adapter->netdev);
	return snmp_fold_field((void **)net->mib.udp_statistics,
			       UDP_MIB_RCVBUFERRORS) +
		snmp_fold_field((void **)net->mib.udplite_statistics,
   				UDP_MIB_RCVBUFERRORS);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
	return snmp_fold_field((void **)udp_statistics,
                               UDP_MIB_RCVBUFERRORS);
#else
        /*
         * XXX udp_statistics isn't exported :(
         * I think we will need to parse the proc node
         */
        return 0;
#endif
}

static void
vmxnet3_drop_checker(unsigned long arg)
{
	struct net_device *netdev = (struct net_device *)arg;
	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
	u64 oob_drops, oob_drops_since_last_check;
	u64 udp_drops, udp_drops_since_last_check;
	u8 oob_percent;
	u64 total_drops;
	u32 current_ring_size;
	Bool growing_ring = FALSE;

current_ring_size = adapter->rx_queue->rx_ring[0].size;

/* get drop statistics */
	oob_drops = vmxnet3_get_oob_drop_count(adapter);
	oob_drops_since_last_check = oob_drops - adapter->prev_oob_drop_count;

udp_drops = vmxnet3_get_udp_drop_count(adapter);
	udp_drops_since_last_check = udp_drops - adapter->prev_udp_drop_count;

total_drops = udp_drops_since_last_check + oob_drops_since_last_check;
	if (total_drops > 0) {
		oob_percent = oob_drops_since_last_check * 100;
		do_div(oob_percent, total_drops);
	} else
		oob_percent = 100;

/* store the current stats to compare next round */
	adapter->prev_oob_drop_count = oob_drops;
	adapter->prev_udp_drop_count = udp_drops;

if(adapter->drop_check_delay > 0) {
           adapter->drop_check_delay--;
           goto reschedule;
        }

if (total_drops > drop_check_noise &&
	    oob_percent > drop_check_min_oob_percent) {
		/* keep track of consecutive intervals of drops */
		adapter->drop_counter++;
		adapter->no_drop_counter = 0;
	}
	else if (total_drops <= drop_check_noise) {
		/* keep track of consecutive intervals of no drops */
		adapter->no_drop_counter++;
	}
	if (oob_drops_since_last_check > current_ring_size &&
	    oob_percent >= drop_check_min_oob_percent) {
		/* if we saw a burst of drops, grow the ring */
		growing_ring = TRUE;
		dev_dbg(&adapter->pdev->dev,
			"%s: detected %llu oob packet drops (%u%% of drops).\n",
			adapter->netdev->name, oob_drops_since_last_check,
			oob_percent);
	} else if (adapter->drop_counter > drop_check_grow_threshold) {
		/* after many intervals with some loss, grow the ring */
		growing_ring = TRUE;
		dev_dbg(&adapter->pdev->dev,
			"%s: reached packet loss threshold, %d ticks\n",
			adapter->netdev->name, adapter->drop_counter);
	}

if (growing_ring &&
            current_ring_size < VMXNET3_RX_RING_MAX_GROWN_SIZE) {
		adapter->new_rx_ring_size =
                   min((u32)VMXNET3_RX_RING_MAX_GROWN_SIZE,
                       current_ring_size*2);
		printk(KERN_INFO "%s: going to grow rx ring.\n",
		       adapter->netdev->name);
		schedule_work(&adapter->resize_ring_work);
                return;
	} else if (adapter->no_drop_counter >= drop_check_shrink_threshold) {
		/*
		 * if we had a lot of intervals with no loss, shrink the
		 * ring again
		 */
		adapter->new_rx_ring_size = current_ring_size / 2;
		if (adapter->new_rx_ring_size < VMXNET3_DEF_RX_RING_SIZE)
			goto reschedule;
		printk(KERN_INFO "%s: going to shrink rx ring.\n",
		       adapter->netdev->name);
		schedule_work(&adapter->resize_ring_work);
                return;
        }
 reschedule:
        /* schedule the next drop check */
        mod_timer(&adapter->drop_check_timer,
                  jiffies + msecs_to_jiffies(drop_check_interval*1000));

}

static void
vmxnet3_start_drop_checker(struct vmxnet3_adapter *adapter)
{
	if (adapter->use_adaptive_ring == FALSE)
		return;

adapter->prev_oob_drop_count = vmxnet3_get_oob_drop_count(adapter);
	adapter->prev_udp_drop_count = vmxnet3_get_udp_drop_count(adapter);
	adapter->drop_counter = 0;
	adapter->no_drop_counter = 0;
        adapter->drop_check_delay = drop_check_delay;
	add_timer(&adapter->drop_check_timer);
}

static void
vmxnet3_stop_drop_checker(struct vmxnet3_adapter *adapter)
{
	del_timer_sync(&adapter->drop_check_timer);
}

/*
 *    allocate and initialize two cmd rings and the completion ring for the
 *    given rx queue. Also allocate and initialize buf_info.
 *    rx buffers are NOT allocated
 */
static int
vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
{
	int i;
	size_t sz;
	struct vmxnet3_rx_buf_info *bi;

BUG_ON(rq->rx_ring[0].size % adapter->rx_buf_per_pkt != 0);
	for (i = 0; i < 2; i++) {
		BUG_ON((rq->rx_ring[i].size & VMXNET3_RING_SIZE_MASK) != 0);
		BUG_ON(rq->rx_ring[i].base != NULL);
		sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
		rq->rx_ring[i].base = pci_alloc_consistent(adapter->pdev, sz,
							&rq->rx_ring[i].basePA);
		if (!rq->rx_ring[i].base) {
			printk(KERN_ERR "%s: failed to allocate rx ring %d\n",
			       adapter->netdev->name, i);
			goto err;
		}
	}

sz = rq->comp_ring.size * sizeof(Vmxnet3_RxCompDesc);
	BUG_ON(rq->comp_ring.base != NULL);
	rq->comp_ring.base = pci_alloc_consistent(adapter->pdev,
			sz,
			&rq->comp_ring.basePA);
	if (!rq->comp_ring.base) {
		printk(KERN_ERR "%s: failed to allocate rx comp ring\n",
		       adapter->netdev->name);
		goto err;
	}

BUG_ON(rq->buf_info[0] || rq->buf_info[1]);
	sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +
						   rq->rx_ring[1].size);
	bi = pci_alloc_consistent(adapter->pdev, sz, &rq->buf_info_pa);
	if (!bi) {
		printk(KERN_ERR "%s: failed to allocate rx bufinfo\n",
		       adapter->netdev->name);
		goto err;
	}
	memset(bi, 0, sz);
	rq->buf_info[0] = bi;
	rq->buf_info[1] = bi + rq->rx_ring[0].size;

return 0;

err:
	vmxnet3_rq_destroy(rq, adapter);
	return -ENOMEM;
}

static int
vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
{
	int i, err = 0;

for (i = 0; i < adapter->num_rx_queues; i++) {
		err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
		if (UNLIKELY(err)) {
			printk(KERN_ERR "%s: failed to create rx queue%i\n",
			       adapter->netdev->name, i);
			goto err_out;
		}
	}
	return err;
err_out:
	vmxnet3_rq_destroy_all(adapter);
	return err;

}

/* Multiple queue aware polling function for tx and rx */
static int
vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
{
	int rcd_done = 0, i;
	if (unlikely(adapter->shared->ecr))
		vmxnet3_process_events(adapter);

for (i = 0; i < adapter->num_tx_queues; i++)
		vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);

for (i = 0; i < adapter->num_rx_queues; i++)
		rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
						   adapter, budget);
	return rcd_done;
}

#ifdef VMXNET3_NAPI
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)

/*
 * New NAPI polling function. Returns # of the NAPI credit consumed (# of rx
 * descriptors processed)
 */
static int
vmxnet3_poll(struct napi_struct *napi, int budget)
{
	struct vmxnet3_rx_queue *rx_queue = container_of(napi,
					  struct vmxnet3_rx_queue, napi);
	int rxd_done;

rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);

if (rxd_done < budget) {
		compat_napi_complete(rx_queue->adapter->netdev, napi);
		vmxnet3_enable_all_intrs(rx_queue->adapter);
	}
	return rxd_done;
}

/*
 * new NAPI polling function for MSI-X mode with multiple Rx queues
 * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
 */

static int
vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
{
	struct vmxnet3_rx_queue *rq = container_of(napi,
						struct vmxnet3_rx_queue, napi);
	struct vmxnet3_adapter *adapter = rq->adapter;
	int rxd_done;

/* When sharing interrupt with corresponding tx queue, process
	 * tx completions in that queue as well
	 */
	if (adapter->share_intr == vmxnet3_intr_buddyshare) {
		struct vmxnet3_tx_queue *tq =
				&adapter->tx_queue[rq - adapter->rx_queue];
		vmxnet3_tq_tx_complete(tq, adapter);
	} else if (adapter->intr.num_intrs == VMXNET3_LINUX_MIN_MSIX_VECT) {
		struct vmxnet3_tx_queue *tq;
		int i;
		for (i = 0; i < adapter->num_tx_queues; i++) {
			tq = &adapter->tx_queue[i];
			vmxnet3_tq_tx_complete(tq, adapter);
		}
	}

rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);

if (rxd_done < budget) {
		compat_napi_complete(adapter->netdev, napi);
		vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
	}
	return rxd_done;
}

#else	/* new NAPI */

/*
 * Result:
 *    0: napi is done
 *    1: continue polling
 */

static int
vmxnet3_poll(struct net_device *poll_dev, int *budget)
{
	int rxd_done, quota;
	struct vmxnet3_adapter *adapter = netdev_priv(poll_dev);

quota = min(*budget, poll_dev->quota);

rxd_done = vmxnet3_do_poll(adapter, quota);

*budget -= rxd_done;
	poll_dev->quota -= rxd_done;

if (rxd_done < quota) {
                compat_napi_complete(poll_dev, &adapter->rx_queue[0].napi);
		vmxnet3_enable_all_intrs(adapter);
		return 0;
	}

return 1; /* not done */
}

/*
 * Poll all rx queues for completions. Does not support napi per rx queue,
 * use one shared napi instead.
 * Returns 0 when napi is done  and 1 when it should continue polling
 */

static int
vmxnet3_poll_rx_only(struct net_device *poll_dev, int *budget)
{
	int rxd_done = 0, quota;
	struct vmxnet3_adapter *adapter = netdev_priv(poll_dev);
	quota = min(*budget, poll_dev->quota);

if (adapter->share_intr == vmxnet3_intr_buddyshare)
		vmxnet3_tq_tx_complete(&adapter->tx_queue[0], adapter);

rxd_done = vmxnet3_rq_rx_complete(&adapter->rx_queue[0],
					  adapter, *budget);
	*budget -= rxd_done;
	poll_dev->quota -= rxd_done;

if (rxd_done < quota) {
		compat_napi_complete(poll_dev, &adapter->rx_queue[0].napi);
			/* enable all rx interrupts*/
		vmxnet3_enable_intr(adapter,
				    adapter->rx_queue[0].comp_ring.intr_idx);
		return 0;
	}

return 1; /* not done */
}

#endif
#endif /* VMXNET3_NAPI  */

#ifdef CONFIG_PCI_MSI

/*
 * Handle completion interrupts on tx queues
 * Returns whether or not the intr is handled
 */

static compat_irqreturn_t
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
vmxnet3_msix_tx(int irq, void *data, struct pt_regs * regs)
#else
vmxnet3_msix_tx(int irq, void *data)
#endif
{
	struct vmxnet3_tx_queue *tq = data;
	struct vmxnet3_adapter *adapter = tq->adapter;

if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
		vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);

/* Handle the case where only one irq is allocate for all tx queues */
	if (adapter->share_intr == vmxnet3_intr_txshare) {
		int i;
		for (i = 0; i < adapter->num_tx_queues; i++) {
			tq = &adapter->tx_queue[i];
			vmxnet3_tq_tx_complete(tq, adapter);
		}
	} else {
		vmxnet3_tq_tx_complete(tq, adapter);
	}
	vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);

return COMPAT_IRQ_HANDLED;
}

/*
 * Handle completion interrupts on rx queues. Returns whether or not the
 * intr is handled
 */

static compat_irqreturn_t
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
vmxnet3_msix_rx(int irq, void *data, struct pt_regs * regs)
#else
vmxnet3_msix_rx(int irq, void *data)
#endif
{
	struct vmxnet3_rx_queue *rq = data;
	struct vmxnet3_adapter *adapter = rq->adapter;

/* disable intr if needed */
	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
		vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
#ifdef VMXNET3_NAPI
	compat_napi_schedule(adapter->netdev, &rq->napi);
#else
	vmxnet3_rq_rx_complete(rq, adapter);
	if (adapter->share_intr == vmxnet3_intr_buddyshare) {
		vmxnet3_tq_tx_complete(&adapter->tx_queue[rq -
				       adapter->rx_queue], adapter);
	} else if (adapter->intr.num_intrs == VMXNET3_LINUX_MIN_MSIX_VECT) {
		struct vmxnet3_tx_queue *tq;
		int i;
		for (i = 0; i < adapter->num_tx_queues; i++) {
			tq = &adapter->tx_queue[i];
			vmxnet3_tq_tx_complete(tq, adapter);
		}
	}
	vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
#endif

return COMPAT_IRQ_HANDLED;
}

/*
 *----------------------------------------------------------------------------
 *
 * vmxnet3_msix_event --
 *
 *    vmxnet3 msix event intr handler
 *
 * Result:
 *    whether or not the intr is handled
 *
 *----------------------------------------------------------------------------
 */

static compat_irqreturn_t
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
vmxnet3_msix_event(int irq, void *data, struct pt_regs * regs)
#else
vmxnet3_msix_event(int irq, void *data)
#endif
{
	struct net_device *dev = data;
	struct vmxnet3_adapter *adapter = netdev_priv(dev);

/* disable intr if needed */
	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
		vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);

if (adapter->shared->ecr)
		vmxnet3_process_events(adapter);

vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);

return COMPAT_IRQ_HANDLED;
}

#endif /* CONFIG_PCI_MSI  */

static compat_irqreturn_t
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
vmxnet3_intr(int irq, void *dev_id, struct pt_regs * regs)
#else
vmxnet3_intr(int irq, void *dev_id)
#endif
{
	struct net_device *dev = dev_id;
	struct vmxnet3_adapter *adapter = netdev_priv(dev);

if (adapter->intr.type == VMXNET3_IT_INTX) {
		u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
		if (unlikely(icr == 0))
			/* not ours */
			return IRQ_NONE;
	}

/* disable intr if needed */
	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
		vmxnet3_disable_all_intrs(adapter);
#ifdef VMXNET3_NAPI
	compat_napi_schedule(dev, &adapter->rx_queue[0].napi);
#else
	vmxnet3_do_poll(adapter, adapter->rx_queue[0].rx_ring[0].size);
	vmxnet3_enable_intr(adapter, 0);
#endif

return COMPAT_IRQ_HANDLED;
}

#ifdef CONFIG_NET_POLL_CONTROLLER
static void
vmxnet3_netpoll(struct net_device *netdev)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);

switch (adapter->intr.type) {
#   ifdef CONFIG_PCI_MSI
	case VMXNET3_IT_MSIX: {
		int i;
		for (i = 0; i < adapter->num_rx_queues; i++) {
#      if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
			vmxnet3_msix_rx(0, &adapter->rx_queue[i], NULL);
#      else
			vmxnet3_msix_rx(0, &adapter->rx_queue[i]);
#      endif
		}
		break;
	}
#   endif	//CONFIG_PCI_MSI
	case VMXNET3_IT_MSI:
	default:
#   if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
		vmxnet3_intr(0, adapter->netdev, NULL);
#   else
		vmxnet3_intr(0, adapter->netdev);
#   endif
		break;
	}
}

#endif		/* CONFIG_NET_POLL_CONTROLLER */

/*
 * event_intr_idx and intr_idx for different comp rings get updated here.
 */

static int
vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
{
	struct vmxnet3_intr *intr = &adapter->intr;
	int err = 0, i;
	int vector = 0;

#ifdef CONFIG_PCI_MSI
	if (adapter->intr.type == VMXNET3_IT_MSIX) {
		for (i = 0; i < adapter->num_tx_queues; i++) {
			sprintf(adapter->tx_queue[i].name, "%s:v%d-%s",
				adapter->netdev->name, vector, "Tx");
			if (adapter->share_intr != vmxnet3_intr_buddyshare)
				err = request_irq(intr->msix_entries[vector].vector,
						  vmxnet3_msix_tx, 0,
						  adapter->tx_queue[i].name,
						  &(adapter->tx_queue[i]));
			if (err) {
				printk(KERN_ERR "Failed to request irq for "
				       "MSIX, %s, error %d\n",
				       adapter->tx_queue[i].name, err);
				return err;
			}

/* Handle the case where only 1 MSIx was allocated for
			 * all tx queues */
			if (adapter->share_intr == vmxnet3_intr_txshare) {
				for (; i < adapter->num_tx_queues; i++)
					adapter->tx_queue[i].comp_ring.intr_idx
								= vector;
				vector++;
				break;
			} else {
				adapter->tx_queue[i].comp_ring.intr_idx
								= vector++;
			}
		}
		if (adapter->share_intr == vmxnet3_intr_buddyshare ||
		    intr->num_intrs == VMXNET3_LINUX_MIN_MSIX_VECT)
			vector = 0;

for (i = 0; i < adapter->num_rx_queues; i++) {
			sprintf(adapter->rx_queue[i].name, "%s:v%d-%s",
				adapter->netdev->name, vector, "Rx");
			err = request_irq(intr->msix_entries[vector].vector,
					  vmxnet3_msix_rx, 0,
					  adapter->rx_queue[i].name,
					  &(adapter->rx_queue[i]));
			if (err) {
				printk(KERN_ERR "Failed to request irq for MSIX"
				       ", %s, error %d\n",
				       adapter->rx_queue[i].name, err);
				return err;
			}

adapter->rx_queue[i].comp_ring.intr_idx = vector++;
		}

sprintf(intr->event_msi_vector_name, "%s:v%d-event",
			adapter->netdev->name, vector);
		err = request_irq(intr->msix_entries[vector].vector,
				  vmxnet3_msix_event, 0,
				  intr->event_msi_vector_name, adapter->netdev);
		intr->event_intr_idx = vector;

} else if (intr->type == VMXNET3_IT_MSI) {
		adapter->num_rx_queues = 1;
		err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
				  adapter->netdev->name, adapter->netdev);
	} else {
#endif
		adapter->num_rx_queues = 1;
		err = request_irq(adapter->pdev->irq, vmxnet3_intr,
				  COMPAT_IRQF_SHARED, adapter->netdev->name,
				  adapter->netdev);
#ifdef CONFIG_PCI_MSI
	}
#endif
	intr->num_intrs = vector + 1;
	if (err) {
		printk(KERN_ERR "Failed to request irq %s (intr type:%d), error"
		       ":%d\n", adapter->netdev->name, intr->type, err);
	} else {
		/* Number of rx queues will not change after this */
		for (i = 0; i < adapter->num_rx_queues; i++) {
			struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
			rq->qid = i;
			rq->qid2 = i + adapter->num_rx_queues;
		}

/* init our intr settings */
		for (i = 0; i < intr->num_intrs; i++)
			intr->mod_levels[i] = UPT1_IML_ADAPTIVE;

if (adapter->intr.type != VMXNET3_IT_MSIX) {
			adapter->intr.event_intr_idx = 0;
			for(i = 0; i < adapter->num_tx_queues; i++) {
				adapter->tx_queue[i].comp_ring.intr_idx = 0;
			}
			adapter->rx_queue[0].comp_ring.intr_idx = 0;
		}

printk(KERN_INFO "%s: intr type %u, mode %u, %u vectors "
		       "allocated\n", adapter->netdev->name, intr->type,
		       intr->mask_mode, intr->num_intrs);
	}

return err;
}

static void
vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
{
	struct vmxnet3_intr *intr = &adapter->intr;
	BUG_ON(intr->type == VMXNET3_IT_AUTO ||
	       intr->num_intrs <= 0);

switch (intr->type) {
#ifdef CONFIG_PCI_MSI
	case VMXNET3_IT_MSIX:
	{
		int i, vector = 0;

if (adapter->share_intr != vmxnet3_intr_buddyshare &&
		    intr->num_intrs != VMXNET3_LINUX_MIN_MSIX_VECT) {
			for (i = 0; i < adapter->num_tx_queues; i++) {
				free_irq(intr->msix_entries[vector++].vector,
					 &(adapter->tx_queue[i]));
				if (adapter->share_intr == vmxnet3_intr_txshare)
					break;
			}
		}

for (i = 0; i < adapter->num_rx_queues; i++) {
			free_irq(intr->msix_entries[vector++].vector,
				 &(adapter->rx_queue[i]));
		}

free_irq(intr->msix_entries[vector].vector,
			 adapter->netdev);
		BUG_ON(vector >= intr->num_intrs);
		break;
	}
	case VMXNET3_IT_MSI:
		free_irq(adapter->pdev->irq, adapter->netdev);
		break;
#endif
	case VMXNET3_IT_INTX:
		free_irq(adapter->pdev->irq, adapter->netdev);
		break;
	default:
		BUG_ON(TRUE);
	}
}

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37)
static void
vmxnet3_vlan_rx_register(struct net_device *netdev, struct vlan_group *grp)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	struct Vmxnet3_DriverShared *shared = adapter->shared;
	u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
	unsigned long flags;

if (grp) {
		/*
                 * VLAN striping feature already enabled by default, no need to
                 * enable it here.
                 */
		if (adapter->netdev->features & NETIF_F_HW_VLAN_RX) {
			int i;
			adapter->vlan_grp = grp;

/*
			 *  Clear entire vfTable; then enable untagged pkts.
			 *  Note: setting one entry in vfTable to non-zero turns
			 *  on VLAN rx filtering.
			 */
			for (i = 0; i < VMXNET3_VFT_SIZE; i++)
				vfTable[i] = 0;

VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
			spin_lock_irqsave(&adapter->cmd_lock, flags);
			VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
					       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
			spin_unlock_irqrestore(&adapter->cmd_lock, flags);
		} else {
			printk(KERN_ERR "%s: vlan_rx_register when device has "
			       "no NETIF_F_HW_VLAN_RX\n", netdev->name);
		}
	} else {
		/* remove vlan rx stripping. */
	        struct Vmxnet3_DSDevRead *devRead = &shared->devRead;

adapter->vlan_grp = NULL;

if (le64_to_cpu(devRead->misc.uptFeatures) & UPT1_F_RXVLAN) {
			int i;

for (i = 0; i < VMXNET3_VFT_SIZE; i++) {
				/* clear entire vfTable; this also disables
				 * VLAN rx filtering
				 */
				vfTable[i] = 0;
			}
			spin_lock_irqsave(&adapter->cmd_lock, flags);
			VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
					       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
			spin_unlock_irqrestore(&adapter->cmd_lock, flags);
		}
	}
}

#endif

static void
vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
{
	u16 vid;
	u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37)
	if (adapter->vlan_grp) {
		Bool activeVlan = FALSE;
		/* continue to allow untagged pkts */
		for (vid = 0; vid < COMPAT_VLAN_GROUP_ARRAY_LEN; vid++) {
			if (compat_vlan_group_get_device(adapter->vlan_grp,
							 vid)) {
				VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
				activeVlan = TRUE;
			}
		}
		if (activeVlan)
			VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
	}
#else
	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
		VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);

/* If any bit is set, set bit 0 to alloca untagged packets */
	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) {
		VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
		break;
	}

#endif
}

/* Inherit net_device features from real device to VLAN device. */
void
vmxnet3_vlan_features(struct vmxnet3_adapter *adapter, u16 vid, Bool allvids)
{
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
	struct net_device *v_netdev;

if (adapter->vlan_grp) {
		if (allvids) {
			for (vid = 0; vid < COMPAT_VLAN_GROUP_ARRAY_LEN; vid++) {
				v_netdev = compat_vlan_group_get_device(
							adapter->vlan_grp, vid);
				if (v_netdev) {
					v_netdev->features |=
						      adapter->netdev->features;
					compat_vlan_group_set_device(
					      adapter->vlan_grp, vid, v_netdev);
				}
			}
		} else {
			v_netdev = compat_vlan_group_get_device(
							adapter->vlan_grp, vid);
			if (v_netdev) {
				v_netdev->features |= adapter->netdev->features;
				compat_vlan_group_set_device(adapter->vlan_grp,
							     vid, v_netdev);
			}
		}
	}
#endif
}

#if COMPAT_LINUX_VERSION_CHECK_LT(3, 3, 0)
static void
#else
static int
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0)
vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
#else
vmxnet3_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
#endif
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	if (!(netdev->flags & IFF_PROMISC)) {
		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
		unsigned long flags;

vmxnet3_vlan_features(adapter, vid, FALSE);
		VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
		spin_lock_irqsave(&adapter->cmd_lock, flags);
		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
	}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)
	set_bit(vid, adapter->active_vlans);
#  if !(COMPAT_LINUX_VERSION_CHECK_LT(3, 3, 0))
	return 0;
#  endif
#endif
}

#if COMPAT_LINUX_VERSION_CHECK_LT(3, 3, 0)
static void
#else
static int
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0)
vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
#else
vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
#endif
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	if (!(netdev->flags & IFF_PROMISC)) {
		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
		unsigned long flags;

VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
		spin_lock_irqsave(&adapter->cmd_lock, flags);
		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
	}

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)
#  if COMPAT_LINUX_VERSION_CHECK_LT(3, 3, 0)
	clear_bit(vid, adapter->active_vlans);
#  else
	clear_bit(vid, adapter->active_vlans);
	return 0;
#  endif
#endif
}

/*
 * Allocate a buffer and copy into the mcast list. Returns NULL if the mcast
 * list exceeds the limit. Returns the addr of the allocated buffer or NULL.
 */

static u8 *
vmxnet3_copy_mc(struct net_device *netdev)
{
	u8 *buf = NULL;
#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 34)
	u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
#else
	u32 sz = netdev->mc_count * ETH_ALEN;
#endif

/* Vmxnet3_RxFilterConf.mfTableLen is u16. */
	if (sz <= 0xffff) {
		/* We may be called with BH disabled */
		buf = kmalloc(sz, GFP_ATOMIC);
		if (buf) {
			int i = 0;
#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 34)
			struct netdev_hw_addr *ha;
			netdev_for_each_mc_addr(ha, netdev)
				memcpy(buf + i++ * ETH_ALEN, ha->addr,
				       ETH_ALEN);
#else
			struct dev_mc_list *mc = netdev->mc_list;
			for (i = 0; i < netdev->mc_count; i++) {
				BUG_ON(!mc);
				memcpy(buf + i * ETH_ALEN, mc->dmi_addr,
				       ETH_ALEN);
				mc = mc->next;
			}
#endif
		}
	}
	return buf;
}

static void
vmxnet3_set_mc(struct net_device *netdev)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	struct Vmxnet3_RxFilterConf *rxConf =
					&adapter->shared->devRead.rxFilterConf;
	unsigned long flags;
	u8 *new_table = NULL;
	dma_addr_t new_table_pa = 0;
	u32 new_mode = VMXNET3_RXM_UCAST;

if (netdev->flags & IFF_PROMISC) {
		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
		memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
		new_mode |= VMXNET3_RXM_PROMISC;
	} else {
		vmxnet3_restore_vlan(adapter);
	}

if (netdev->flags & IFF_BROADCAST)
		new_mode |= VMXNET3_RXM_BCAST;

if (netdev->flags & IFF_ALLMULTI)
		new_mode |= VMXNET3_RXM_ALL_MULTI;
#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 34)
	else if (netdev_mc_count(netdev) > 0) {
#else
	else if (netdev->mc_count > 0) {
#endif
		new_table = vmxnet3_copy_mc(netdev);
		if (new_table) {
			new_mode |= VMXNET3_RXM_MCAST;
			rxConf->mfTableLen = cpu_to_le16(
#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 34)
					netdev_mc_count(netdev) * ETH_ALEN);
#else
					netdev->mc_count * ETH_ALEN);
#endif

new_table_pa = pci_map_single(adapter->pdev,
						      new_table,
						      rxConf->mfTableLen,
						      PCI_DMA_TODEVICE);
			rxConf->mfTablePA = cpu_to_le64(new_table_pa);
		} else {
			printk(KERN_INFO "%s: failed to copy mcast list, "
			       "setting ALL_MULTI\n", netdev->name);
			new_mode |= VMXNET3_RXM_ALL_MULTI;
		}
	}

if (!(new_mode & VMXNET3_RXM_MCAST)) {
		rxConf->mfTableLen = 0;
		rxConf->mfTablePA = 0;
	}

spin_lock_irqsave(&adapter->cmd_lock, flags);
	if (new_mode != rxConf->rxMode) {
		rxConf->rxMode = cpu_to_le32(new_mode);
		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
				       VMXNET3_CMD_UPDATE_RX_MODE);
		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
	}

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
			       VMXNET3_CMD_UPDATE_MAC_FILTERS);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);

if (new_table) {
		pci_unmap_single(adapter->pdev, new_table_pa,
				 rxConf->mfTableLen, PCI_DMA_TODEVICE);
		kfree(new_table);
	}
}

/*
 * Wipes out the whole driver_shared area and re-initializes it
 */

static void
vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
{
	struct Vmxnet3_DriverShared *shared = adapter->shared;
	struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
	struct Vmxnet3_TxQueueConf *tqc;
	struct Vmxnet3_RxQueueConf *rqc;
	int i;

memset(shared, 0, sizeof(*shared));

/* driver settings */
	shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
	devRead->misc.driverInfo.version = cpu_to_le32(
						VMXNET3_DRIVER_VERSION_NUM);
	devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
				VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
	devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
	*((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
				*((u32 *)&devRead->misc.driverInfo.gos));
	devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
	devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);

devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
	devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));

/* set up feature flags */
	if (adapter->rxcsum)
		set_flag_le64(&devRead->misc.uptFeatures, UPT1_F_RXCSUM);

if (adapter->lro) {
		set_flag_le64(&devRead->misc.uptFeatures, UPT1_F_LRO);
		devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
	}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0)
	if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
#else
	if (adapter->netdev->features & NETIF_F_HW_VLAN_RX) {
#endif
                 /*
                  * Event there is no VLAN enabled,
                  * VLAN Tag stripping is enabled by default. This is
                  * required to work around  a Palo bug.
                  */
		set_flag_le64(&devRead->misc.uptFeatures, UPT1_F_RXVLAN);
	}

devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
	devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
	devRead->misc.queueDescLen = cpu_to_le32(
		adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
		adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));

/* tx queue settings */
	devRead->misc.numTxQueues =  adapter->num_tx_queues;
	for(i = 0; i < adapter->num_tx_queues; i++) {
		struct vmxnet3_tx_queue	*tq = &adapter->tx_queue[i];
		BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
		tqc = &adapter->tqd_start[i].conf;
		tqc->txRingBasePA   = cpu_to_le64(tq->tx_ring.basePA);
		tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
		tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
		tqc->ddPA           = cpu_to_le64(tq->buf_info_pa);
		tqc->txRingSize     = cpu_to_le32(tq->tx_ring.size);
		tqc->dataRingSize   = cpu_to_le32(tq->data_ring.size);
		tqc->compRingSize   = cpu_to_le32(tq->comp_ring.size);
		tqc->ddLen          = cpu_to_le32(sizeof(struct vmxnet3_tx_buf_info) *
				      tqc->txRingSize);
		tqc->intrIdx        = tq->comp_ring.intr_idx;
	}

/* rx queue settings */
	devRead->misc.numRxQueues = adapter->num_rx_queues;
	for(i = 0; i < adapter->num_rx_queues; i++) {
		struct vmxnet3_rx_queue	*rq = &adapter->rx_queue[i];
		rqc = &adapter->rqd_start[i].conf;
		rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
		rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
		rqc->compRingBasePA  = cpu_to_le64(rq->comp_ring.basePA);
		rqc->ddPA            = cpu_to_le64(rq->buf_info_pa);
		rqc->rxRingSize[0]   = cpu_to_le32(rq->rx_ring[0].size);
		rqc->rxRingSize[1]   = cpu_to_le32(rq->rx_ring[1].size);
		rqc->compRingSize    = cpu_to_le32(rq->comp_ring.size);
		rqc->ddLen           = cpu_to_le32(sizeof(struct vmxnet3_rx_buf_info) *
				     (rqc->rxRingSize[0] + rqc->rxRingSize[1]));
		rqc->intrIdx         = rq->comp_ring.intr_idx;
	}

#ifdef VMXNET3_RSS
	memset(adapter->rss_conf, 0, sizeof (*adapter->rss_conf));

if (adapter->rss) {
		UPT1_RSSConf *rssConf = adapter->rss_conf;
		devRead->misc.uptFeatures |= UPT1_F_RSS;
		devRead->misc.numRxQueues = adapter->num_rx_queues;
		rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
				    UPT1_RSS_HASH_TYPE_IPV4 |
				    UPT1_RSS_HASH_TYPE_TCP_IPV6 |
				    UPT1_RSS_HASH_TYPE_IPV6;
		rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
		rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
		rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
		get_random_bytes(&rssConf->hashKey[0], rssConf->hashKeySize);
		if (num_rss_entries >= adapter->dev_number *
				       VMXNET3_RSS_IND_TABLE_SIZE) {
			int j = (adapter->dev_number) *
				VMXNET3_RSS_IND_TABLE_SIZE;
			for (i = 0; i < rssConf->indTableSize; i++, j++) {
				if (rss_ind_table[j] >= 0 &&
				    rss_ind_table[j] < adapter->num_rx_queues)
					rssConf->indTable[i] = rss_ind_table[j];
				else
					rssConf->indTable[i] =
						compat_ethtool_rxfh_indir_default(i,
						adapter->num_rx_queues);
			}
		} else {
			for (i = 0; i < rssConf->indTableSize; i++)
				rssConf->indTable[i] =
					compat_ethtool_rxfh_indir_default(i,
					adapter->num_rx_queues);
		}

printk(KERN_INFO "RSS indirection table :\n");
		for (i = 0; i < rssConf->indTableSize; i++)
			printk("%2d ", rssConf->indTable[i]);
		printk("\n");

devRead->rssConfDesc.confVer = 1;
		devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
		devRead->rssConfDesc.confPA  = cpu_to_le64(adapter->rss_conf_pa);
	}

#endif /* VMXNET3_RSS */

/* intr settings */
	devRead->intrConf.autoMask = adapter->intr.mask_mode ==
				     VMXNET3_IMM_AUTO;
	devRead->intrConf.numIntrs = adapter->intr.num_intrs;
	for (i = 0; i < adapter->intr.num_intrs; i++)
		devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];

devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
        devRead->intrConf.intrCtrl |= VMXNET3_IC_DISABLE_ALL;

/* rx filter settings */
	devRead->rxFilterConf.rxMode = 0;
	vmxnet3_restore_vlan(adapter);
	vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
	/* the rest are already zeroed */
}

/*
 * put the vNIC into an operational state. After this function finishes, the
 * adapter is fully functional. It does the following:
 * 1. initialize tq and rq
 * 2. fill rx rings with rx buffers
 * 3. setup intr
 * 4. setup driver_shared
 * 5. activate the dev
 * 6. signal the stack that the vNIC is ready to tx/rx
 * 7. enable intrs for the vNIC
 *
 * Returns:
 *    0 if the vNIC is in operation state
 *    error code if any intermediate step fails.
 */

int
vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
{
	int err, i;
	u32 ret;
	unsigned long flags;

dev_dbg(&adapter->pdev->dev, "%s: skb_buf_size %d, rx_buf_per_pkt %d, "
		"ring sizes %u %u %u\n", adapter->netdev->name,
		adapter->skb_buf_size, adapter->rx_buf_per_pkt,
		adapter->tx_queue[0].tx_ring.size,
		adapter->rx_queue[0].rx_ring[0].size,
		adapter->rx_queue[0].rx_ring[1].size);

vmxnet3_tq_init_all(adapter);
	err = vmxnet3_rq_init_all(adapter);
	if (err) {
		printk(KERN_ERR "Failed to init rx queue for %s: error %d\n",
		       adapter->netdev->name, err);
		goto rq_err;
	}

err = vmxnet3_request_irqs(adapter);
	if (err) {
		printk(KERN_ERR "Failed to setup irq for %s: error %d\n",
		       adapter->netdev->name, err);
		goto irq_err;
	}

vmxnet3_setup_driver_shared(adapter);

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL,VMXNET3_GET_ADDR_LO(
			       adapter->shared_pa));
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
			       adapter->shared_pa));

spin_lock_irqsave(&adapter->cmd_lock, flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
			       VMXNET3_CMD_ACTIVATE_DEV);
	ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
	if (ret != 0) {
		printk(KERN_ERR "Failed to activate dev %s: error %u\n",
		       adapter->netdev->name, ret);
		err = -EINVAL;
		goto activate_err;
	}

for (i = 0; i < adapter->num_rx_queues; i++) {
		VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD +
				(i * VMXNET3_REG_ALIGN)),
				adapter->rx_queue[i].rx_ring[0].next2fill);
		VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +
				(i * VMXNET3_REG_ALIGN)),
				adapter->rx_queue[i].rx_ring[1].next2fill);
	}

/* Apply the rx filter settins last. */
	vmxnet3_set_mc(adapter->netdev);

/*
	 * Check link state when first activating device. It will start the
	 * tx queue if the link is up.
	 */
	vmxnet3_check_link(adapter, TRUE);
#ifdef VMXNET3_NAPI
	{
		int i;
		for(i = 0; i < adapter->num_rx_queues; i++)
			compat_napi_enable(adapter->netdev,
					   &adapter->rx_queue[i].napi);
	}
#endif
	vmxnet3_enable_all_intrs(adapter);
	vmxnet3_start_drop_checker(adapter);
	clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
	return 0;

activate_err:
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
	vmxnet3_free_irqs(adapter);
irq_err:
rq_err:
	/* free up buffers we allocated */
	vmxnet3_rq_cleanup_all(adapter);
	return err;
}

void
vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
{
	unsigned long flags;

spin_lock_irqsave(&adapter->cmd_lock, flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
}

/*
 * Stop the device. After this function returns, the adapter stop pkt tx/rx
 * and won't generate intrs. The stack won't try to xmit pkts through us,
 * nor will it poll us for pkts. It does the following:
 *
 * 1. ask the vNIC to quiesce
 * 2. disable the vNIC from generating intrs
 * 3. free intr
 * 4. stop the stack from xmiting pkts thru us and polling
 * 5. free rx buffers
 * 6. tx complete pkts pending
 *
 */

int
vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
{
	unsigned long flags;

if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
		return 0;

spin_lock_irqsave(&adapter->cmd_lock, flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
			       VMXNET3_CMD_QUIESCE_DEV);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
	vmxnet3_stop_drop_checker(adapter);
	vmxnet3_disable_all_intrs(adapter);
#ifdef VMXNET3_NAPI
	{
		int i;
		for(i = 0; i < adapter->num_rx_queues; i++)
			compat_napi_disable(adapter->netdev,
					    &adapter->rx_queue[i].napi);
	}
#endif
	netif_tx_disable(adapter->netdev);
	adapter->link_speed = 0;
	netif_carrier_off(adapter->netdev);

vmxnet3_tq_cleanup_all(adapter);
	vmxnet3_rq_cleanup_all(adapter);
	vmxnet3_free_irqs(adapter);
	return 0;
}

static void
vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
{
	u32 tmp;

tmp = *(u32 *)mac;
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);

tmp = (mac[5] << 8) | mac[4];
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
}

static int
vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
{
	struct sockaddr *addr = p;
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);

memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
	vmxnet3_write_mac_addr(adapter, netdev->dev_addr);

return 0;
}

/* ==================== initialization and cleanup routines ============ */

static int
vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, Bool *dma64)
{
	int err;
	unsigned long mmio_start, mmio_len;
	struct pci_dev *pdev = adapter->pdev;

err = pci_enable_device(pdev);
	if (err) {
		printk(KERN_ERR "Failed to enable adapter %s: error %d\n",
		       pci_name(pdev), err);
		return err;
	}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 6)
	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
		if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
			printk(KERN_ERR "pci_set_consistent_dma_mask failed "
			       "for adapter %s\n", pci_name(pdev));
			err = -EIO;
			goto err_set_mask;
		}
		*dma64 = TRUE;
	} else {
		if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
			printk(KERN_ERR "pci_set_dma_mask failed for adapter "
			       "%s\n", pci_name(pdev));
			err = -EIO;
			goto err_set_mask;
		}
		*dma64 = FALSE;
	}
#else
	*dma64 = TRUE;
#endif

err = pci_request_regions(pdev, vmxnet3_driver_name);
	if (err) {
		printk(KERN_ERR "Failed to request region for adapter %s: "
		       "error %d\n", pci_name(pdev), err);
		goto err_set_mask;
	}

pci_set_master(pdev);

mmio_start = pci_resource_start(pdev, 0);
	mmio_len = pci_resource_len(pdev, 0);
	adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
	if (!adapter->hw_addr0) {
		printk(KERN_ERR "Failed to map bar0 for adapter %s\n",
		       pci_name(pdev));
		err = -EIO;
		goto err_ioremap;
	}

mmio_start = pci_resource_start(pdev, 1);
	mmio_len = pci_resource_len(pdev, 1);
	adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
	if (!adapter->hw_addr1) {
		printk(KERN_ERR "Failed to map bar1 for adapter %s\n",
		       pci_name(pdev));
		err = -EIO;
		goto err_bar1;
	}
	return 0;

err_bar1:
	iounmap(adapter->hw_addr0);
err_ioremap:
	pci_release_regions(pdev);
err_set_mask:
	pci_disable_device(pdev);
	return err;
}

static void
vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
{
	BUG_ON(!adapter->pdev);

iounmap(adapter->hw_addr0);
	iounmap(adapter->hw_addr1);
	pci_release_regions(adapter->pdev);
	pci_disable_device(adapter->pdev);
}

/*
 * Calculate the # of buffers for a pkt based on mtu, then adjust the size of
 * the 1st rx ring accordingly
 */

static void
vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
{
	size_t sz, i, ring0_size, ring1_size, comp_size;
	struct vmxnet3_rx_queue	*rq = &adapter->rx_queue[0];

if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
				    VMXNET3_MAX_ETH_HDR_SIZE) {
		adapter->skb_buf_size = adapter->netdev->mtu +
					VMXNET3_MAX_ETH_HDR_SIZE;
		if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
			adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;

adapter->rx_buf_per_pkt = 1;
	} else {
		adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
		sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
					    VMXNET3_MAX_ETH_HDR_SIZE;
		adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
	}

if (adapter->is_shm) {
		adapter->skb_buf_size = PAGE_SIZE;
	}

/*
	 * for simplicity, force the ring0 size to be a multiple of
	 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
	 */
	sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
	ring0_size = adapter->rx_queue[0].rx_ring[0].size;
	ring0_size = (ring0_size + sz - 1) / sz * sz;
	ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
			   sz * sz);
	ring1_size = adapter->rx_queue[0].rx_ring[1].size;
	comp_size = ring0_size + ring1_size;

for(i = 0; i < adapter->num_rx_queues; i++) {
		rq = &adapter->rx_queue[i];
		rq->rx_ring[0].size = ring0_size;
		rq->rx_ring[1].size = ring1_size;
		rq->comp_ring.size = comp_size;
	}
}

/* Create the specified number of tx queues and rx queues. On failure, it
 * destroys the queues created. */
int
vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
		      u32 rx_ring_size, u32 rx_ring2_size)
{
	int err = 0, i;

for (i = 0; i<adapter->num_tx_queues; i++) {
		struct vmxnet3_tx_queue	*tq = &adapter->tx_queue[i];
		tq->tx_ring.size   = tx_ring_size;
		tq->data_ring.size = tx_ring_size;
		tq->comp_ring.size = tx_ring_size;
		tq->shared = &adapter->tqd_start[i].ctrl;
		tq->stopped = TRUE;
		tq->adapter = adapter;
		tq->qid = i;
		err = vmxnet3_tq_create(tq, adapter);
		/*
		 * Too late to change num_tx_queues. We cannot do away with
		 * lesser number of queues than what we asked for
		 */
		if (err)
			goto queue_err;
	}

adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
	adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
	vmxnet3_adjust_rx_ring_size(adapter);
	for (i = 0; i<adapter->num_rx_queues; i++) {
		struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
		/* qid and qid2 for rx queues will be assigned later when num
		 * of rx queues is finalized after allocating intrs */
		rq->shared = &adapter->rqd_start[i].ctrl;
		rq->adapter = adapter;
		err = vmxnet3_rq_create(rq, adapter);
		if (err) {
			if (i == 0) {
				printk(KERN_ERR "Could not allocate any rx"
				       "queues. Aborting.\n");
				goto queue_err;
			} else {
				printk(KERN_INFO "Number of rx queues changed "
				       "to : %d. \n", i);
				adapter->num_rx_queues = i;
				err = 0;
				break;
			}
		}
	}
	return err;
queue_err:
	vmxnet3_tq_destroy_all(adapter);
	return err;
}

/*
 * setup rings, allocate necessary resources, request for IRQs, configure
 * the device. The device is functional after this function finishes
 * successfully.
 * Returns 0 on success, negative errno value on failure
 */

static int
vmxnet3_open(struct net_device *netdev)
{
	struct vmxnet3_adapter *adapter;
	int err, i;
	uint32 flags;
	unsigned long irq_flags;

netif_carrier_off(netdev);
	adapter = compat_netdev_priv(netdev);

for (i = 0; i < adapter->num_tx_queues; i++)
		spin_lock_init(&adapter->tx_queue[i].tx_lock);

if (adapter->is_shm) {
		printk(KERN_INFO "bringing up shared memory vmxnet3 %s\n",
		       netdev->name);
		err = vmxnet3_shm_open(adapter, netdev->name, shm_pool_size);
		if (err) {
			goto shm_err;
		}
	}

err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
				    VMXNET3_DEF_RX_RING_SIZE,
				    VMXNET3_DEF_RX_RING_SIZE);
	if (err)
		goto queue_err;

spin_lock_irqsave(&adapter->cmd_lock, irq_flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
			       VMXNET3_CMD_GET_ADAPTIVE_RING_INFO);
	flags = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
	spin_unlock_irqrestore(&adapter->cmd_lock, irq_flags);
	adapter->use_adaptive_ring = !(flags & VMXNET3_DISABLE_ADAPTIVE_RING);
	compat_setup_timer(&adapter->drop_check_timer, vmxnet3_drop_checker,
                           (unsigned long)netdev);
	adapter->drop_check_timer.expires = jiffies +
		msecs_to_jiffies(drop_check_interval * 1000);

err = vmxnet3_activate_dev(adapter);
	if (err)
		goto activate_err;

COMPAT_NETDEV_MOD_INC_USE_COUNT;

return 0;

activate_err:
	vmxnet3_rq_destroy_all(adapter);
	vmxnet3_tq_destroy_all(adapter);
queue_err:
	if (adapter->is_shm) {
		vmxnet3_shm_close(adapter);
	}
shm_err:
	return err;
}

static int
vmxnet3_close(struct net_device *netdev)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);

/*
	 * Reset_work may be in the middle of resetting the device, wait for its
	 * completion.
	 */
	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
		compat_msleep(1);

vmxnet3_quiesce_dev(adapter);

if (adapter->is_shm) {
		vmxnet3_shm_close(adapter);
	}

vmxnet3_rq_destroy_all(adapter);
	vmxnet3_tq_destroy_all(adapter);

COMPAT_NETDEV_MOD_DEC_USE_COUNT;

clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);

return 0;
}

/*
 * Called to forcibly close the device when the driver failed to re-activate it.
 */
void
vmxnet3_force_close(struct vmxnet3_adapter *adapter)
{
	/*
	 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
	 * vmxnet3_close() will deadlock.
	 */
	BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
#ifdef VMXNET3_NAPI
	{
		/* We need to enable NAPI, otherwise dev_close will deadlock.
		 * dev_close leads us to napi_disable which loops until "polling
		 * scheduled" bit is set. This bit is cleared by napi_enable()
		 */
		int i;
		for (i = 0; i < adapter->num_rx_queues; i++)
			compat_napi_enable(adapter->netdev,
					   &adapter->rx_queue[i].napi);
	}
#endif	/* VMXNET3_NAPI */
	dev_close(adapter->netdev);
}

int
vmxnet3_set_ringsize(struct net_device *netdev, u32 new_tx_ring_size,
                     u32 new_rx_ring_size, u32 new_rx_ring2_size)
{
	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
	int err = 0;

if (new_tx_ring_size == adapter->tx_queue[0].tx_ring.size &&
	    new_rx_ring_size == adapter->rx_queue[0].rx_ring[0].size &&
            new_rx_ring2_size == adapter->rx_queue[0].rx_ring[1].size) {
		return 0;
	}

/*
	 * Reset_work may be in the middle of resetting the device, wait for its
	 * completion.
	 */
	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
		compat_msleep(1);

if (netif_running(netdev)) {
		vmxnet3_quiesce_dev(adapter);
		vmxnet3_reset_dev(adapter);

/* recreate the rx queue and the tx queue based on the
		 * new sizes */
		vmxnet3_tq_destroy_all(adapter);
		vmxnet3_rq_destroy_all(adapter);

err = vmxnet3_create_queues(adapter, new_tx_ring_size,
					    new_rx_ring_size,
                                            new_rx_ring2_size);

if (err) {
			/* failed, most likely because of OOM, try the default
			 * size */
			printk(KERN_ERR "%s: failed to apply new sizes, try the"
			        "default ones\n", netdev->name);
			err = vmxnet3_create_queues(adapter,
						    VMXNET3_DEF_TX_RING_SIZE,
						    VMXNET3_DEF_RX_RING_SIZE,
						    VMXNET3_DEF_RX_RING_SIZE);
			if (err) {
				printk(KERN_ERR "%s: failed to create queues "
				       "with default sizes. Closing it\n",
				       netdev->name);
				goto out;
			}
		}

err = vmxnet3_activate_dev(adapter);
		if (err)
			printk(KERN_ERR "%s: failed to re-activate, error %d."
			       " Closing it\n", netdev->name, err);
	}

out:
	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
	if (err)
		vmxnet3_force_close(adapter);

return err;
}

/* Some distros support 68 as minimum MTU. IP header can be upto 60 bytes
 * long (20 + 40) and the minimum fragment length is 8 bytes. Trying to set lesser
 * renders the interface useless.
 */
#define MIN_MTU 68

static int
vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	int err = 0;

if (new_mtu < MIN_MTU || new_mtu > VMXNET3_MAX_MTU)
		return -EINVAL;

netdev->mtu = new_mtu;

/*
	 * Reset_work may be in the middle of resetting the device, wait for its
	 * completion.
	 */
	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
		compat_msleep(1);

if (netif_running(netdev)) {
		vmxnet3_quiesce_dev(adapter);
		vmxnet3_reset_dev(adapter);

/* we need to re-create the rx queue based on the new mtu */
		vmxnet3_rq_destroy_all(adapter);
		vmxnet3_adjust_rx_ring_size(adapter);
		err = vmxnet3_rq_create_all(adapter);
		if (err) {
			printk(KERN_ERR "%s: failed to re-create rx queues,"
				" error %d. Closing it.\n", netdev->name, err);
			goto out;
		}

err = vmxnet3_activate_dev(adapter);
		if (err) {
			printk(KERN_ERR "%s: failed to re-activate, error %d. "
				"Closing it\n", netdev->name, err);
			goto out;
		}
	}

out:
	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
	if (err)
		vmxnet3_force_close(adapter);

return err;
}

static void
vmxnet3_declare_features(struct vmxnet3_adapter *adapter, Bool dma64)
{
	struct net_device *netdev = adapter->netdev;

printk(KERN_INFO "features: sg csum vlan jf tso");

adapter->rxcsum = TRUE;

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
	 /* LRO feature control by module param */
	if (!disable_lro) {
#else
	{
		/* LRO feature control by ethtool */
		netdev->features |= NETIF_F_LRO;
#endif

adapter->lro = TRUE;
		printk(" lro");
	}

#ifdef NETIF_F_TSO6
	netdev->features |= NETIF_F_TSO6;
	printk(" tsoIPv6");
#endif

if (dma64) {
		netdev->features |= NETIF_F_HIGHDMA;
		printk(" highDMA");
	}

#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0)
	netdev->hw_features = netdev->features & ~NETIF_F_HW_VLAN_CTAG_FILTER;
	netdev->vlan_features = netdev->hw_features & ~(NETIF_F_HW_VLAN_CTAG_TX |
							NETIF_F_HW_VLAN_CTAG_RX);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
	netdev->hw_features = netdev->features & ~NETIF_F_HW_VLAN_FILTER;
	netdev->vlan_features = netdev->hw_features & ~(NETIF_F_HW_VLAN_TX |
							NETIF_F_HW_VLAN_RX);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
	netdev->vlan_features = netdev->features;
#endif
	printk("\n");

}

static void
vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
{
	u32 tmp;

tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
	*(u32 *)mac = tmp;

tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
	mac[4] = tmp & 0xff;
	mac[5] = (tmp >> 8) & 0xff;
}

#ifdef CONFIG_PCI_MSI
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)

/*
 * Enable MSIx vectors.
 * Returns :
 *	0 when number of vectors enabled is between requested and the minimum
 *	    required (VMXNET3_LINUX_MIN_MSIX_VECT),
 *	number of vectors which can be enabled otherwise (this number is smaller
 *	    than VMXNET3_LINUX_MIN_MSIX_VECT)
 */

static int
vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter,
                             int vectors)
{
	int err = 0, vector_threshold;
	vector_threshold = VMXNET3_LINUX_MIN_MSIX_VECT;

while (vectors >= vector_threshold) {
		err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,
				      vectors);
		if (!err) {
			adapter->intr.num_intrs = vectors;
			return 0;
		} else if (err < 0) {
			printk(KERN_ERR "Failed to enable MSI-X for %s, error"
			       " %d\n",	adapter->netdev->name, err);
			vectors = 0;
		} else if (err < vector_threshold) {
			break;
		} else {
			/* If fails to enable required number of MSI-x vectors
			 * try enabling minimum number of vectors required.
			 */
			vectors = vector_threshold;
			printk(KERN_ERR "Failed to enable %d MSI-X for %s, try"
			       " %d instead\n", vectors, adapter->netdev->name,
			       vector_threshold);
		}
	}

printk(KERN_INFO "Number of MSI-X interrupts which can be allocated are "
	  "lower than min threshold required.\n");
   return err;
}

#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19) */
#endif /* CONFIG_PCI_MSI */

/*
 * read the intr configuration, pick the intr type, and enable MSI/MSI-X if
 * needed. adapter->intr.{type, mask_mode, num_intr} are modified
 */

static void
vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
{
	u32 cfg;
	unsigned long flags;

/* intr settings */
	spin_lock_irqsave(&adapter->cmd_lock, flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
			       VMXNET3_CMD_GET_CONF_INTR);
	cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
	adapter->intr.type = cfg & 0x3;
	adapter->intr.mask_mode = (cfg >> 2) & 0x3;

#ifdef CONFIG_PCI_MSI
	if (adapter->intr.type == VMXNET3_IT_AUTO) {
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
		/* start with MSI-X */
		adapter->intr.type = VMXNET3_IT_MSIX;
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 13)
		adapter->intr.type = VMXNET3_IT_MSI;
#else
		adapter->intr.type = VMXNET3_IT_INTX;
#endif
	}

if (adapter->intr.type == VMXNET3_IT_MSIX) {
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
		int vector, err = 0;
		/* start with MSI-X */
		adapter->intr.num_intrs = (adapter->share_intr ==
					   vmxnet3_intr_txshare) ? 1 :
					   adapter->num_tx_queues;
		adapter->intr.num_intrs += (adapter->share_intr ==
					   vmxnet3_intr_buddyshare) ? 0 :
					   adapter->num_rx_queues;
		adapter->intr.num_intrs += 1;		/* for link event */

adapter->intr.num_intrs = (adapter->intr.num_intrs >
					   VMXNET3_LINUX_MIN_MSIX_VECT
					   ? adapter->intr.num_intrs :
					   VMXNET3_LINUX_MIN_MSIX_VECT);

for (vector = 0; vector < adapter->intr.num_intrs; vector++)
			adapter->intr.msix_entries[vector].entry = vector;

err = vmxnet3_acquire_msix_vectors(adapter,
						   adapter->intr.num_intrs);
		/* If we cannot allocate one MSIx vector per queue
		 * then limit the number of rx queues to 1
		 */
		if(err == VMXNET3_LINUX_MIN_MSIX_VECT) {
			if (adapter->share_intr != vmxnet3_intr_buddyshare
			    || adapter->num_rx_queues != 1) {
				adapter->share_intr = vmxnet3_intr_txshare;
				printk(KERN_ERR "Number of rx queues : 1\n");
				adapter->num_rx_queues = 1;
				adapter->intr.num_intrs =
						VMXNET3_LINUX_MIN_MSIX_VECT;
			}
			return;
		}
		if (!err)
			return;

/* If we cannot allocate MSIx vectors use only one rx queue */
		printk(KERN_INFO "Failed to enable MSI-X for %s, error %d."
		       "#rx queues : 1, try MSI\n", adapter->netdev->name, err);
#else
		printk(KERN_INFO "MSI-X not supported in kernels earlier than"
		       "2.6.19. Trying to use MSI instead...\n");
#endif
		adapter->intr.type = VMXNET3_IT_MSI;
	}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 13)
	if (adapter->intr.type == VMXNET3_IT_MSI) {
		if (!pci_enable_msi(adapter->pdev)) {
			adapter->num_rx_queues = 1;
			adapter->intr.num_intrs = 1;
			return;
		}
	}
#endif
#endif		/* CONFIG_MSI */
	adapter->num_rx_queues = 1;
	printk(KERN_INFO "Using INTx interrupt\n");
	adapter->intr.type = VMXNET3_IT_INTX;

/* INT-X related setting */
	adapter->intr.num_intrs = 1;
}

static void
vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
{
#ifdef CONFIG_PCI_MSI
	if (adapter->intr.type == VMXNET3_IT_MSIX)
		pci_disable_msix(adapter->pdev);
	else if (adapter->intr.type == VMXNET3_IT_MSI)
		pci_disable_msi(adapter->pdev);
	else
#endif
	{
		BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
	}

}

/*
 * Called when the stack detects a Tx hang. Schedule a job to reset the device
 */

static void
vmxnet3_tx_timeout(struct net_device *netdev)
{
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	adapter->tx_timeout_count++;

printk(KERN_ERR "%s: tx hang\n", adapter->netdev->name);
	compat_schedule_work(&adapter->reset_work);
}

static void
vmxnet3_resize_ring_work(compat_work_arg data)
{
	struct vmxnet3_adapter *adapter;
	adapter = COMPAT_WORK_GET_DATA(data, struct vmxnet3_adapter,
                                       resize_ring_work);

(void)vmxnet3_set_ringsize(adapter->netdev,
				   adapter->tx_queue[0].tx_ring.size,
				   adapter->new_rx_ring_size,
                                   adapter->rx_queue[0].rx_ring[1].size);
}

static void
vmxnet3_reset_work(compat_work_arg data)
{
	struct vmxnet3_adapter *adapter;

adapter = COMPAT_WORK_GET_DATA(data, struct vmxnet3_adapter,
				       reset_work);

/* if another thread is resetting the device, no need to proceed */
	if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
		return;

/* if the device is closed or is being opened, we must leave it alone */
	if (netif_running(adapter->netdev)  &&
	    (adapter->netdev->flags & IFF_UP)) {
		printk(KERN_INFO "%s: resetting\n", adapter->netdev->name);

vmxnet3_quiesce_dev(adapter);
		vmxnet3_reset_dev(adapter);
		vmxnet3_activate_dev(adapter);
	} else {
		printk(KERN_INFO "%s: already closed\n", adapter->netdev->name);
	}

clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
}

/*
 * Initialize a vmxnet3 device. Returns 0 on success, negative errno code
 * otherwise. Initialize the h/w and allocate necessary resources
 */

static int
vmxnet3_probe_device(struct pci_dev *pdev,
		     const struct pci_device_id *id)
{
#ifdef HAVE_NET_DEVICE_OPS
	static const struct net_device_ops vmxnet3_netdev_ops = {
		.ndo_open = vmxnet3_open,
		.ndo_stop = vmxnet3_close,
		.ndo_start_xmit = vmxnet3_xmit_frame,
		.ndo_set_mac_address = vmxnet3_set_mac_addr,
		.ndo_change_mtu = vmxnet3_change_mtu,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
		.ndo_set_features = vmxnet3_set_features,
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 36)
		.ndo_get_stats64 = vmxnet3_get_stats64,
#else
		.ndo_get_stats = vmxnet3_get_stats,
#endif
		.ndo_tx_timeout = vmxnet3_tx_timeout,
#if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
		.ndo_set_multicast_list = vmxnet3_set_mc,
#else
		.ndo_set_rx_mode = vmxnet3_set_mc,
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 37)
		.ndo_vlan_rx_register = vmxnet3_vlan_rx_register,
#endif
		.ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
		.ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
#ifdef CONFIG_NET_POLL_CONTROLLER
		.ndo_poll_controller = vmxnet3_netpoll,
#endif
	};
#endif	/* HAVE_NET_DEVICE_OPS  */
	int err;
	Bool dma64 = FALSE; /* stupid gcc */
	u32 ver;
	struct net_device *netdev;
	struct vmxnet3_adapter *adapter;
	u8 mac[ETH_ALEN];
	int size;
	int num_tx_queues, num_rx_queues;
	int dev_number = devices_found;

#ifdef VMXNET3_RSS
	if ((dev_number < VMXNET3_SHM_MAX_DEVICES &&
	    enable_shm[dev_number] == 1 && correct_shm_disclaimer) ||
	    !compat_multiqueue_allowed(pdev))
		num_rx_queues = 1;
	else {
		num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
				(int)num_online_cpus());
		if (num_rqs[dev_number] > 0) {
			num_rx_queues = min(num_rx_queues, num_rqs[dev_number]);
		}
	}

if (!compat_is_power_of_2(num_rx_queues))
		num_rx_queues = compat_rounddown_pow_of_two(num_rx_queues);
#else
	num_rx_queues = 1;
#endif
	/* CONFIG_NETDEVICES_MULTIQUEUE dictates if skb has queue_mapping field.
	 * It was introduced in 2.6.23 onwards but Ubuntu 804 (2.6.24) and
	 * SLES 11(2.6.25) do not have it.
	 */

#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 25) || defined(CONFIG_NETDEVICES_MULTIQUEUE)
	if ((dev_number < VMXNET3_SHM_MAX_DEVICES &&
	    enable_shm[dev_number] == 1 && correct_shm_disclaimer) ||
            !compat_multiqueue_allowed(pdev))
		num_tx_queues = 1;
	else {
		num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
				(int)num_online_cpus());
		if (num_tqs[dev_number] > 0) {
			num_tx_queues = min(num_tx_queues, num_tqs[dev_number]);
		}
	}

if (!compat_is_power_of_2(num_tx_queues))
		num_tx_queues = compat_rounddown_pow_of_two(num_tx_queues);

netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
				   num_tx_queues);
#else
	num_tx_queues = 1;
	netdev = compat_alloc_etherdev(sizeof(struct vmxnet3_adapter));
#endif
	if (!netdev) {
		printk(KERN_ERR "Failed to alloc ethernet device %s\n",
			pci_name(pdev));
		return -ENOMEM;
	}

pci_set_drvdata(pdev, netdev);
	adapter = compat_netdev_priv(netdev);
	adapter->netdev = netdev;
	adapter->pdev = pdev;
	spin_lock_init(&adapter->cmd_lock);
	adapter->adapter_pa = pci_map_single(adapter->pdev, adapter,
					     sizeof(struct vmxnet3_adapter),
					     PCI_DMA_TODEVICE);
	adapter->shared = pci_alloc_consistent(adapter->pdev,
			  sizeof(struct Vmxnet3_DriverShared),
			  &adapter->shared_pa);
	if (!adapter->shared) {
		printk(KERN_ERR "Failed to allocate memory for %s\n",
			pci_name(pdev));
		err = -ENOMEM;
		goto err_alloc_shared;
	}

adapter->num_rx_queues = num_rx_queues;
	adapter->num_tx_queues = num_tx_queues;
	adapter->rx_buf_per_pkt = 1;

adapter->pm_conf = pci_alloc_consistent(adapter->pdev,
						sizeof(struct Vmxnet3_PMConf),
						&adapter->pm_conf_pa);
	if (adapter->pm_conf == NULL) {
		printk(KERN_ERR "Failed to allocate memory for %s\n",
			pci_name(pdev));
		err = -ENOMEM;
		goto err_alloc_pm;
	}

#ifdef VMXNET3_RSS

adapter->rss_conf = pci_alloc_consistent(adapter->pdev,
						 sizeof(struct UPT1_RSSConf),
						 &adapter->rss_conf_pa);
	if (adapter->rss_conf == NULL) {
		printk(KERN_ERR "Failed to allocate memory for %s\n",
		       pci_name(pdev));
		err = -ENOMEM;
		goto err_alloc_rss;
	}
#endif /* VMXNET3_RSS */

err = vmxnet3_alloc_pci_resources(adapter, &dma64);
	if (err < 0)
		goto err_alloc_pci;

ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
	if (ver & 2) {
		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 2);
		adapter->version = 2;
	} else if (ver & 1) {
		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
		adapter->version = 1;
	} else {
		printk(KERN_ERR "Incompatible h/w version (0x%x) for adapter"
				" %s\n", ver, pci_name(pdev));
		err = -EBUSY;
		goto err_ver;
	}

ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
	if (ver & 1) {
		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
	} else {
		printk(KERN_ERR "Incompatible upt version (0x%x) for "
		       "adapter %s\n", ver, pci_name(pdev));
		err = -EBUSY;
		goto err_ver;
	}

vmxnet3_declare_features(adapter, dma64);

adapter->dev_number = dev_number;
	adapter->is_shm = FALSE;
	if (adapter->dev_number < VMXNET3_SHM_MAX_DEVICES) {
		if (enable_shm[adapter->dev_number] == 1) {
			if (!correct_shm_disclaimer) {
				printk(KERN_ERR "Did not activate shm, "
				       "disclaimer missing\n");
			} else {
				adapter->is_shm = TRUE;
			}
		}
	}

/*
	 * Sharing intr between corresponding tx and rx queues gets priority
	 * over all tx queues sharing an intr. Also, to use buddy interrupts
	 * number of tx queues should be same as number of rx queues.
	 */
	if (share_tx_intr[adapter->dev_number] == 1)
		adapter->share_intr = vmxnet3_intr_txshare;
	else if (buddy_intr[adapter->dev_number] == 1 &&
		 adapter->num_tx_queues == adapter->num_rx_queues)
		adapter->share_intr = vmxnet3_intr_buddyshare;
	else
		adapter->share_intr = vmxnet3_intr_noshare;

vmxnet3_alloc_intr_resources(adapter);

size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
	size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
	adapter->tqd_start = pci_alloc_consistent(adapter->pdev, size,
			     &adapter->queue_desc_pa);

if (!adapter->tqd_start) {
		printk(KERN_ERR "Failed to allocate memory for %s\n",
			pci_name(pdev));
		err = -ENOMEM;
		goto err_alloc_queue_desc;
	}
	adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
                                                        adapter->num_tx_queues);

printk(KERN_INFO "# of Tx queues : %d, # of Rx queues : %d\n",
	       adapter->num_tx_queues, adapter->num_rx_queues);

#ifdef VMXNET3_RSS
	if (adapter->num_rx_queues > 1 &&
	    adapter->intr.type == VMXNET3_IT_MSIX) {
		adapter->rss = TRUE;
		printk("RSS is enabled.\n");
	} else {
		adapter->rss = FALSE;
	}
#endif
	vmxnet3_read_mac_addr(adapter, mac);
	memcpy(netdev->dev_addr,  mac, netdev->addr_len);
#ifdef ETHTOOL_GPERMADDR
	memcpy(netdev->perm_addr, mac, netdev->addr_len);
#endif
#ifdef HAVE_NET_DEVICE_OPS
	netdev->netdev_ops = &vmxnet3_netdev_ops;
#else
	netdev->open  = vmxnet3_open;
	netdev->stop  = vmxnet3_close;
	netdev->hard_start_xmit = vmxnet3_xmit_frame;
	netdev->set_mac_address = vmxnet3_set_mac_addr;
	netdev->change_mtu = vmxnet3_change_mtu;
	netdev->get_stats = vmxnet3_get_stats;
	netdev->tx_timeout = vmxnet3_tx_timeout;
	netdev->set_multicast_list = vmxnet3_set_mc;
	netdev->vlan_rx_register = vmxnet3_vlan_rx_register;
	netdev->vlan_rx_add_vid  = vmxnet3_vlan_rx_add_vid;
	netdev->vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid;

#ifdef CONFIG_NET_POLL_CONTROLLER
	netdev->poll_controller  = vmxnet3_netpoll;
#endif
#endif /* HAVE_NET_DEVICE_OPS  */
	netdev->watchdog_timeo = 5 * HZ;
	vmxnet3_set_ethtool_ops(netdev);

COMPAT_INIT_WORK(&adapter->reset_work, vmxnet3_reset_work, adapter);
	COMPAT_INIT_WORK(&adapter->resize_ring_work,
			 vmxnet3_resize_ring_work, adapter);
	set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);

#ifdef VMXNET3_NAPI
	if (adapter->intr.type == VMXNET3_IT_MSIX) {
		int i;
		for (i = 0; i < adapter->num_rx_queues; i++) {
			compat_netif_napi_add(netdev,
					      &adapter->rx_queue[i].napi,
					      vmxnet3_poll_rx_only, 64);
		}
	} else {
		compat_netif_napi_add(netdev, &adapter->rx_queue[0].napi,
				      vmxnet3_poll, 64);
	}

#endif

COMPAT_SET_MODULE_OWNER(netdev);
	COMPAT_SET_NETDEV_DEV(netdev, &pdev->dev);

err = register_netdev(netdev);
	if (err) {
		printk(KERN_ERR "Failed to register adapter %s\n",
		       pci_name(pdev));
		goto err_register;
	}

vmxnet3_check_link(adapter, FALSE);
	devices_found++;
	return 0;

err_register:
	pci_free_consistent(adapter->pdev, size, adapter->tqd_start,
			    adapter->queue_desc_pa);
err_alloc_queue_desc:
	vmxnet3_free_intr_resources(adapter);
err_ver:
	vmxnet3_free_pci_resources(adapter);
err_alloc_pci:
#ifdef VMXNET3_RSS
	pci_free_consistent(adapter->pdev, sizeof(struct UPT1_RSSConf),
			    adapter->rss_conf, adapter->rss_conf_pa);
err_alloc_rss:
#endif
	pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_PMConf),
			    adapter->pm_conf, adapter->pm_conf_pa);
err_alloc_pm:
	pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
			    adapter->shared, adapter->shared_pa);
err_alloc_shared:
	pci_unmap_single(adapter->pdev, adapter->adapter_pa,
			 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
	pci_set_drvdata(pdev, NULL);
	compat_free_netdev(netdev);
	return err;
}

static void
vmxnet3_remove_device(struct pci_dev *pdev)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	int size = 0;

compat_flush_scheduled_work(&adapter->reset_work);
	compat_flush_scheduled_work(&adapter->resize_ring_work);

unregister_netdev(netdev);

vmxnet3_free_intr_resources(adapter);
	vmxnet3_free_pci_resources(adapter);
#ifdef VMXNET3_RSS
	pci_free_consistent(adapter->pdev, sizeof(struct UPT1_RSSConf),
			    adapter->rss_conf, adapter->rss_conf_pa);
#endif
	pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_PMConf),
			    adapter->pm_conf, adapter->pm_conf_pa);

size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
	size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
	pci_free_consistent(adapter->pdev, size, adapter->tqd_start,
			    adapter->queue_desc_pa);
	pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared),
			    adapter->shared, adapter->shared_pa);
	pci_unmap_single(adapter->pdev, adapter->adapter_pa,
			 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
	compat_free_netdev(netdev);
}

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 12)

static void vmxnet3_shutdown_device(struct pci_dev *pdev)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	unsigned long flags;

/*
	 * Reset_work may be in the middle of resetting the device, wait for its
	 * completion.
	 */
	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
		compat_msleep(1);

if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED,
		    &adapter->state)) {
			clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
			return;
		}
	spin_lock_irqsave(&adapter->cmd_lock, flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
			       VMXNET3_CMD_QUIESCE_DEV);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
	vmxnet3_stop_drop_checker(adapter);
	vmxnet3_disable_all_intrs(adapter);

clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
}

#endif

#ifdef CONFIG_PM

/*
 *      May programs the wake-up filters if configured to do so.
 */

static int
vmxnet3_suspend(struct pci_dev *pdev, pm_message_t state)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	struct Vmxnet3_PMConf *pmConf;
	struct ethhdr *ehdr;
	struct arphdr *ahdr;
	u8 *arpreq;
	struct in_device *in_dev;
	struct in_ifaddr *ifa;
	int i = 0;
	unsigned long flags;

if (!netif_running(netdev))
		return 0;

vmxnet3_stop_drop_checker(adapter);
#ifdef VMXNET3_NAPI
	{
		int j;
		for(j = 0; j < adapter->num_rx_queues; j++)
			compat_napi_disable(adapter->netdev,
					    &adapter->rx_queue[j].napi);
	}
#endif
	vmxnet3_disable_all_intrs(adapter);
	vmxnet3_free_irqs(adapter);
	vmxnet3_free_intr_resources(adapter);
	netif_device_detach(netdev);
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
	netif_stop_queue(netdev);
#else
	netif_tx_stop_all_queues(netdev);
#endif

/* Create wake-up filters. */
	pmConf = adapter->pm_conf;
	memset(pmConf, 0, sizeof(*pmConf));

if (adapter->wol & WAKE_UCAST) {
		pmConf->filters[i].patternSize = ETH_ALEN;
		pmConf->filters[i].maskSize = 1;
		memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
		pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */

set_flag_le16(&pmConf->wakeUpEvents, VMXNET3_PM_WAKEUP_FILTER);
		i++;
	}

if (adapter->wol & WAKE_ARP) {
		in_dev = in_dev_get(netdev);
		if (!in_dev)
			goto skip_arp;

ifa = (struct in_ifaddr *)in_dev->ifa_list;
		if (!ifa) {
			dev_dbg(&adapter->pdev->dev, "Cannot program WoL ARP"
				" filter for %s: no IPv4 address.\n",
				netdev->name);
			in_dev_put(in_dev);
			goto skip_arp;
		}
		pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header */
			sizeof(struct arphdr) +                  /* ARP header */
			2 * ETH_ALEN +                           /* 2 Ethernet addresses */
			2 * sizeof (u32);                     /* 2 IPv4 addresses */
		pmConf->filters[i].maskSize =
			(pmConf->filters[i].patternSize - 1) / 8 + 1;
		/* ETH_P_ARP in Ethernet header. */
		ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
		ehdr->h_proto = htons(ETH_P_ARP);
		/* ARPOP_REQUEST in ARP header. */
		ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
		ahdr->ar_op = htons(ARPOP_REQUEST);
		arpreq = (u8 *)(ahdr + 1);

/* The Unicast IPv4 address in 'tip' field. */
		arpreq += 2 * ETH_ALEN + sizeof(u32);
		*(u32 *)arpreq = ifa->ifa_address;

/* The mask for the relevant bits. */
		pmConf->filters[i].mask[0] = 0x00;
		pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
		pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
		pmConf->filters[i].mask[3] = 0x00;
		pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
		pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
		in_dev_put(in_dev);

set_flag_le16(&pmConf->wakeUpEvents, VMXNET3_PM_WAKEUP_FILTER);
		i++;
	}

skip_arp:
	if (adapter->wol & WAKE_MAGIC)
		set_flag_le16(&pmConf->wakeUpEvents, VMXNET3_PM_WAKEUP_MAGIC);

pmConf->numFilters = i;

adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
	adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
								  *pmConf));
	adapter->shared->devRead.pmConfDesc.confPA =
		cpu_to_le64(adapter->pm_conf_pa);

spin_lock_irqsave(&adapter->cmd_lock, flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
			       VMXNET3_CMD_UPDATE_PMCFG);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);

compat_pci_save_state(pdev);
	pci_enable_wake(pdev, compat_pci_choose_state(pdev, state),
			adapter->wol);
	pci_disable_device(pdev);
	pci_set_power_state(pdev, compat_pci_choose_state(pdev, state));

return 0;
}

static int
vmxnet3_resume(struct pci_dev *pdev)
{
	int err;
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct vmxnet3_adapter *adapter = compat_netdev_priv(netdev);
	unsigned long flags;

if (!netif_running(netdev))
		return 0;

pci_set_power_state(pdev, PCI_D0);
	compat_pci_restore_state(pdev);
	err = pci_enable_device(pdev);
	if (err != 0)
		goto err;

pci_enable_wake(pdev, PCI_D0, 0);

vmxnet3_alloc_intr_resources(adapter);

/*
	 * During hibernate and suspend, device has to be reinitialized
	 * as the device state need not be preserved.
	 */

/*
	 * Other reset tasks cannot run during resume hence
	 * not checking adapter state.
	 */
	spin_lock_irqsave(&adapter->cmd_lock, flags);
	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
			       VMXNET3_CMD_QUIESCE_DEV);
	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
	vmxnet3_tq_cleanup_all(adapter);
	vmxnet3_rq_cleanup_all(adapter);

vmxnet3_reset_dev(adapter);
	err = vmxnet3_activate_dev(adapter);
	if (err){
		printk(KERN_ERR "%s: failed to re-activate on resume, error %d."
			       " Closing it\n", netdev->name, err);
		vmxnet3_force_close(adapter);
		goto err;
	}
	netif_device_attach(netdev);

err:
	return err;
}

#endif

static struct pci_driver vmxnet3_driver = {
	.name		= vmxnet3_driver_name,
	.id_table	= vmxnet3_pciid_table,
	.probe		= vmxnet3_probe_device,
	.remove		= vmxnet3_remove_device,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 12)
	.shutdown	= vmxnet3_shutdown_device,
#endif
#ifdef CONFIG_PM
	.suspend	= vmxnet3_suspend,
	.resume		= vmxnet3_resume,
#endif
};

static int __init
vmxnet3_init_module(void)
{
	printk(KERN_INFO "%s - version %s\n", VMXNET3_DRIVER_DESC,
	       VMXNET3_DRIVER_VERSION_REPORT);

devices_found = 0;
	correct_shm_disclaimer = shm_disclaimer &&
		(strncmp(shm_disclaimer, VMXNET3_SHM_DISCLAIMER,
		strlen(VMXNET3_SHM_DISCLAIMER)) == 0);

#ifdef CONFIG_COMPAT
#ifndef HAVE_UNLOCKED_IOCTL
	if (correct_shm_disclaimer) {
		register_ioctl32_conversion(SHM_IOCTL_TX, NULL);
		register_ioctl32_conversion(SHM_IOCTL_ALLOC_ONE, NULL);
		register_ioctl32_conversion(SHM_IOCTL_FREE_ONE, NULL);
	}
#endif
#endif
	return pci_register_driver(&vmxnet3_driver);
}

module_init(vmxnet3_init_module);

static void
vmxnet3_exit_module(void)
{
#ifdef CONFIG_COMPAT
#ifndef HAVE_UNLOCKED_IOCTL
	if (correct_shm_disclaimer) {
		unregister_ioctl32_conversion(SHM_IOCTL_TX);
		unregister_ioctl32_conversion(SHM_IOCTL_ALLOC_ONE);
		unregister_ioctl32_conversion(SHM_IOCTL_FREE_ONE);
	}
#endif
#endif
	pci_unregister_driver(&vmxnet3_driver);
}

module_exit(vmxnet3_exit_module);

MODULE_AUTHOR("VMware, Inc.");
MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
MODULE_LICENSE("GPL v2");
MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);
/*
 * Starting with SLE10sp2, Novell requires that IHVs sign a support agreement
 * with them and mark their kernel modules as externally supported via a
 * change to the module header. If this isn't done, the module will not load
 * by default (i.e., neither mkinitrd nor modprobe will accept it).
 */
MODULE_INFO(supported, "external");
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 24)
module_param(disable_lro, int, 0);
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 2)
MODULE_PARM(enable_shm, "0-10i");
#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 10)
module_param_array(enable_shm, int, num_enable_shm, 0);
#else
module_param_array(enable_shm, int, &num_enable_shm, 0);
#endif
MODULE_PARM_DESC(enable_shm, "Shared memory enable");
module_param(shm_disclaimer, charp, 0);
MODULE_PARM_DESC(shm_disclaimer, "Shared memory disclaimer");
module_param(shm_pool_size, int, 0);
MODULE_PARM_DESC(shm_pool_size, "Shared memory pool size");

#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 25) || \
    defined(CONFIG_NETDEVICES_MULTIQUEUE)
static unsigned int num_adapters = 0;
module_param_array(share_tx_intr, int, &num_adapters, 0);
MODULE_PARM_DESC(share_tx_intr, "Share an intr among all tx completions. "
		 "Comma separated list of 1s and 0s - one for each NIC. "
		 "1 to share, 0 to not, default is 0");
module_param_array(buddy_intr, int, &num_adapters, 0);
MODULE_PARM_DESC(buddy_intr, "Share an intr among corresponding tx and rx "
		 "queues. Comma separated list of 1s and 0s - one for each "
		 "NIC. 1 to share, 0 to not, default is 1");
module_param_array(num_tqs, int, &num_adapters, 0);
MODULE_PARM_DESC(num_tqs, "Number of transmit queues in each adapter. Comma "
		 "separated list of ints. Default is 0 which makes number"
		 " of queues same as number of CPUs");
#endif
#ifdef VMXNET3_RSS
module_param_array(rss_ind_table, int, &num_rss_entries, 0);
MODULE_PARM_DESC(rss_ind_table, "RSS Indirection table. Number of entries "
		 "per NIC should be 32. Each comma separated entry is a rx "
		 "queue number starting with 0. Repeat the same for all NICs");
module_param_array(num_rqs, int, &num_adapters, 0);
MODULE_PARM_DESC(num_rqs, "Number of receive queues in each adapter. Comma "
		 " separated list of ints. Default is 0 which makes number"
		 " of queues same as number of CPUs");

#endif /* VMXNET3_RSS */
module_param(drop_check_noise, uint, 0);
MODULE_PARM_DESC(drop_check_noise, "Number of drops per interval which are ignored");
module_param(drop_check_grow_threshold, uint, 0);
MODULE_PARM_DESC(drop_check_shrink_threshold, "Threshold for growing the ring");
��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/vmxnet3_shm_shared.h�������������������������������������������������������������������0000444�0000000�0000000�00000005542�13432725332�016121� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2009 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmxnet3_shm_shared.h --
 *
 *   Header shared between vmxnet3 shared memory kernel driver and userspace.
 *
 */

#ifndef __VMXNET_SHARED_SHM
#define __VMXNET_SHARED_SHM

#include <linux/ioctl.h>

// ioctl constants
#define SHM_IOCTL_MAGIC 'v'
#define SHM_IOCTL_TX                  _IO(SHM_IOCTL_MAGIC, 0)
#define SHM_IOCTL_ALLOC_ONE           _IO(SHM_IOCTL_MAGIC, 1)
#define SHM_IOCTL_FREE_ONE            _IO(SHM_IOCTL_MAGIC, 4)

/*
 * invalid index
 *
 * Must be 0 so that a invalid shared memory page has the same
 * value as a NULL struct page. We need that because we overload
 * the same field for regular and shared memory version of vmxnet3.
 */
#define SHM_INVALID_IDX 0

// sizes of shared memory regions in pages
#define SHM_DATA_START 0
#define SHM_DEFAULT_DATA_SIZE 4096
#define SHM_MAX_DATA_SIZE 16384
#define SHM_CTL_START 16384
#define SHM_CTL_SIZE 1

// ring size (in entries) is limited by the single control page - 4 bytes per re
#define SHM_RX_RING_SIZE 500
#define SHM_TX_RING_SIZE 500

// maximum fragments per packet is 16 (64k) + 2 for metadata
#define VMXNET3_SHM_MAX_FRAGS 18

// shared memory ring entry
struct vmxnet3_shm_ringentry
{
   uint16_t idx;      // index of this page in the pool
   uint16_t len: 13;  // length of data in this page
   uint16_t own: 1;   // whether the receiver owns the re
   uint16_t eop: 1;   // end of packet
   uint16_t trash: 1; // ignore all the data in this packet, but still take ownership
};

static const struct vmxnet3_shm_ringentry RE_ZERO = {0,0,0,0,0};

// shared memory control page
struct vmxnet3_shm_ctl
{
   struct vmxnet3_shm_ringentry rx_ring[SHM_RX_RING_SIZE];
   struct vmxnet3_shm_ringentry tx_ring[SHM_TX_RING_SIZE];

// XXX move kernel_* into the kernel, currently here for debugging
   // user_rxi is used by poll() to avoid going to sleep when there are packets waiting
   uint16_t user_rxi, user_txi;
   uint16_t kernel_rxi, kernel_txi;

struct
   {
      uint64_t user_rx, user_tx;
      uint64_t kernel_rx, kernel_tx;
   } stats;

uint64_t channelBad;
};

/*
 * vmxnet3_version.h --
 *
 * Version definitions for the Linux vmxnet3 driver.
 */

#ifndef _VMXNET3_VERSION_H_
#define _VMXNET3_VERSION_H_

#define VMXNET3_DRIVER_VERSION          1.4.3.0
#define VMXNET3_DRIVER_VERSION_COMMAS   1,4,3,0
#define VMXNET3_DRIVER_VERSION_STRING   "1.4.3.0"

#define VMXNET3_DRIVER_VERSION_NUM      0x01040300

#endif /* _VMXNET3_VERSION_H_ */
��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/��������������������������������������������������������������������������������0000755�0000000�0000000�00000000000�13432726376�013422� 5����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_statfs.h�����������������������������������������������������������������0000444�0000000�0000000�00000002306�13432725347�016436� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_STATFS_H__
#   define __COMPAT_STATFS_H__

/* vfs.h simply include statfs.h, but it knows what directory statfs.h is in. */
#include <linux/vfs.h>

/* 2.5.74 renamed struct statfs to kstatfs. */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 74)
#define compat_kstatfs kstatfs
#else
#define compat_kstatfs statfs
#endif

#endif /* __COMPAT_STATFS_H__ */
��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vmciKernelAPI.h�����������������������������������������������������������������0000444�0000000�0000000�00000002451�13432725350�016213� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2010,2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmciKernelAPI.h --
 *
 *    Kernel API (current) exported from the VMCI host and guest drivers.
 */

#ifndef __VMCI_KERNELAPI_H__
#define __VMCI_KERNELAPI_H__

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#include "includeCheck.h"

/* With this file you always get the latest version. */
#include "vmciKernelAPI1.h"
#include "vmciKernelAPI2.h"
#include "vmciKernelAPI3.h"

#endif /* !__VMCI_KERNELAPI_H__ */

�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/community_source.h��������������������������������������������������������������0000444�0000000�0000000�00000003712�13432725350�017167� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2009-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * community_source.h --
 *
 *    Macros for excluding source code from community.
 */

#ifndef _COMMUNITY_SOURCE_H_
#define _COMMUNITY_SOURCE_H_

#define INCLUDE_ALLOW_USERLEVEL

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_VMKDRIVERS
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMCORE
#include "includeCheck.h"

/* 
 * Convenience macro for COMMUNITY_SOURCE
 */
#undef EXCLUDE_COMMUNITY_SOURCE
#ifdef COMMUNITY_SOURCE
   #define EXCLUDE_COMMUNITY_SOURCE(x) 
#else
   #define EXCLUDE_COMMUNITY_SOURCE(x) x
#endif

#undef COMMUNITY_SOURCE_AMD_SECRET
#if !defined(COMMUNITY_SOURCE) || defined(AMD_SOURCE)
/*
 * It's ok to include AMD_SECRET source code for non-Community Source,
 * or for drops directed at AMD.
 */
   #define COMMUNITY_SOURCE_AMD_SECRET
#endif

#undef COMMUNITY_SOURCE_INTEL_SECRET
#if !defined(COMMUNITY_SOURCE) || defined(INTEL_SOURCE)
/*
 * It's ok to include INTEL_SECRET source code for non-Community Source,
 * or for drops directed at Intel.
 */
   #define COMMUNITY_SOURCE_INTEL_SECRET
#endif

#endif
������������������������������������������������������vmxnet3-only/shared/vmci_iocontrols.h���������������������������������������������������������������0000444�0000000�0000000�00000062156�13432725350�017003� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007-2014,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmci_iocontrols.h
 *
 *        The VMCI driver io controls.
 */

#ifndef _VMCI_IOCONTROLS_H_
#define _VMCI_IOCONTROLS_H_

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#include "includeCheck.h"

#include "vm_assert.h"
#include "vmci_defs.h"

#if defined(_WIN32) && defined(WINNT_DDK)
/* We need to expose the API through an IOCTL on Windows.  Use latest API. */
#include "vmciKernelAPI.h"
#endif // _WIN32 && WINNT_DDK

#if defined __cplusplus
extern "C" {
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIVA64ToPtr --
 *
 *      Convert a VA64 to a pointer.
 *
 * Results:
 *      Virtual address.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void *
VMCIVA64ToPtr(VA64 va64) // IN
{
#ifdef VM_64BIT
   ASSERT_ON_COMPILE(sizeof (void *) == 8);
#else
   ASSERT_ON_COMPILE(sizeof (void *) == 4);
   // Check that nothing of value will be lost.
   ASSERT(!(va64 >> 32));
#endif
   return (void *)(uintptr_t)va64;
}

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIPtrToVA64 --
 *
 *      Convert a pointer to a VA64.
 *
 * Results:
 *      Virtual address.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE VA64
VMCIPtrToVA64(void const *ptr) // IN
{
   ASSERT_ON_COMPILE(sizeof ptr <= sizeof (VA64));
   return (VA64)(uintptr_t)ptr;
}

/*
 * Driver version.
 *
 * Increment major version when you make an incompatible change.
 * Compatibility goes both ways (old driver with new executable
 * as well as new driver with old executable).
 */

#define VMCI_VERSION_SHIFT_WIDTH   16 /* Never change this. */
#define VMCI_MAKE_VERSION(_major, _minor)    ((_major) <<                \
                                              VMCI_VERSION_SHIFT_WIDTH | \
                                              (uint16) (_minor))
#define VMCI_VERSION_MAJOR(v)  ((uint32) (v) >> VMCI_VERSION_SHIFT_WIDTH)
#define VMCI_VERSION_MINOR(v)  ((uint16) (v))

/*
 * VMCI_VERSION is always the current version.  Subsequently listed
 * versions are ways of detecting previous versions of the connecting
 * application (i.e., VMX).
 *
 * VMCI_VERSION_NOVMVM: This version removed support for VM to VM
 * communication.
 *
 * VMCI_VERSION_NOTIFY: This version introduced doorbell notification
 * support.
 *
 * VMCI_VERSION_HOSTQP: This version introduced host end point support
 * for hosted products.
 *
 * VMCI_VERSION_PREHOSTQP: This is the version prior to the adoption of
 * support for host end-points.
 *
 * VMCI_VERSION_PREVERS2: This fictional version number is intended to
 * represent the version of a VMX which doesn't call into the driver
 * with ioctl VERSION2 and thus doesn't establish its version with the
 * driver.
 */

#define VMCI_VERSION                VMCI_VERSION_NOVMVM
#define VMCI_VERSION_NOVMVM         VMCI_MAKE_VERSION(11, 0)
#define VMCI_VERSION_NOTIFY         VMCI_MAKE_VERSION(10, 0)
#define VMCI_VERSION_HOSTQP         VMCI_MAKE_VERSION(9, 0)
#define VMCI_VERSION_PREHOSTQP      VMCI_MAKE_VERSION(8, 0)
#define VMCI_VERSION_PREVERS2       VMCI_MAKE_VERSION(1, 0)

/*
 * VMCISockets driver version.  The version is platform-dependent and is
 * embedded in vsock_version.h for each platform.  It can be obtained via
 * VMCISock_Version() (which uses IOCTL_VMCI_SOCKETS_VERSION).  The
 * following is simply for constructing an unsigned integer value from the
 * comma-separated version in the header.  This must match the macros defined
 * in vmci_sockets.h.  An example of using this is:
 * uint16 parts[4] = { VSOCK_DRIVER_VERSION_COMMAS };
 * uint32 version = VMCI_SOCKETS_MAKE_VERSION(parts);
 */

#define VMCI_SOCKETS_MAKE_VERSION(_p) \
   ((((_p)[0] & 0xFF) << 24) | (((_p)[1] & 0xFF) << 16) | ((_p)[2]))

#if defined(__linux__) || defined(VMKERNEL)
/*
 * Linux defines _IO* macros, but the core kernel code ignore the encoded
 * ioctl value. It is up to individual drivers to decode the value (for
 * example to look at the size of a structure to determine which version
 * of a specific command should be used) or not (which is what we
 * currently do, so right now the ioctl value for a given command is the
 * command itself).
 *
 * Hence, we just define the IOCTL_VMCI_foo values directly, with no
 * intermediate IOCTLCMD_ representation.
 */
#  define IOCTLCMD(_cmd) IOCTL_VMCI_ ## _cmd
#elif defined (__APPLE__)
#include <sys/ioccom.h>
#define IOCTLCMD(_cmd) IOCTL_VMCI_ ## _cmd
#define IOCTLCMD_I(_cmd, _type) \
   IOCTL_VMCI_MACOS_ ## _cmd = _IOW('V', IOCTL_VMCI_ ## _cmd, _type)
#define IOCTLCMD_O(_cmd, _type) \
   IOCTL_VMCI_MACOS_ ## _cmd = _IOR('V', IOCTL_VMCI_ ## _cmd, _type)
#define IOCTLCMD_IO(_cmd, _type) \
   IOCTL_VMCI_MACOS_ ## _cmd = _IOWR('V', IOCTL_VMCI_ ## _cmd, _type)
#else // if defined(__linux__)
/*
 * On platforms other than Linux, IOCTLCMD_foo values are just numbers, and
 * we build the IOCTL_VMCI_foo values around these using platform-specific
 * format for encoding arguments and sizes.
 */
#  define IOCTLCMD(_cmd) IOCTLCMD_VMCI_ ## _cmd
#endif

enum IOCTLCmd_VMCI {
   /*
    * We need to bracket the range of values used for ioctls, because x86_64
    * Linux forces us to explicitly register ioctl handlers by value for
    * handling 32 bit ioctl syscalls.  Hence FIRST and LAST.  Pick something
    * for FIRST that doesn't collide with vmmon (2001+).
    */
#if defined(__linux__)
   IOCTLCMD(FIRST) = 1951,
#else
   /* Start at 0. */
   IOCTLCMD(FIRST),
#endif
   IOCTLCMD(VERSION) = IOCTLCMD(FIRST),

/* BEGIN VMCI */
   IOCTLCMD(INIT_CONTEXT),

/*
    * The following two were used for process and datagram process creation.
    * They are not used anymore and reserved for future use.
    * They will fail if issued.
    */
   IOCTLCMD(RESERVED1),
   IOCTLCMD(RESERVED2),

/*
    * The following used to be for shared memory. It is now unused and and is
    * reserved for future use. It will fail if issued.
    */
   IOCTLCMD(RESERVED3),

/*
    * The follwoing three were also used to be for shared memory. An
    * old WS6 user-mode client might try to use them with the new
    * driver, but since we ensure that only contexts created by VMX'en
    * of the appropriate version (VMCI_VERSION_NOTIFY or
    * VMCI_VERSION_NEWQP) or higher use these ioctl, everything is
    * fine.
    */
   IOCTLCMD(QUEUEPAIR_SETVA),
   IOCTLCMD(NOTIFY_RESOURCE),
   IOCTLCMD(NOTIFICATIONS_RECEIVE),
   IOCTLCMD(VERSION2),
   IOCTLCMD(QUEUEPAIR_ALLOC),
   IOCTLCMD(QUEUEPAIR_SETPAGEFILE),
   IOCTLCMD(QUEUEPAIR_DETACH),
   IOCTLCMD(DATAGRAM_SEND),
   IOCTLCMD(DATAGRAM_RECEIVE),
   IOCTLCMD(DATAGRAM_REQUEST_MAP),
   IOCTLCMD(DATAGRAM_REMOVE_MAP),
   IOCTLCMD(CTX_ADD_NOTIFICATION),
   IOCTLCMD(CTX_REMOVE_NOTIFICATION),
   IOCTLCMD(CTX_GET_CPT_STATE),
   IOCTLCMD(CTX_SET_CPT_STATE),
   IOCTLCMD(GET_CONTEXT_ID),
   /* END VMCI */

/*
    * BEGIN VMCI SOCKETS
    *
    * We mark the end of the vmci commands and the start of the vmci sockets
    * commands since they are used in separate modules on Linux.
    * */
   IOCTLCMD(LAST),
   IOCTLCMD(SOCKETS_FIRST) = IOCTLCMD(LAST),

/*
    * This used to be for accept() on Windows and Mac OS, which is now
    * redundant (since we now use real handles).  It is used instead for
    * getting the version.  This value is now public, so it cannot change.
    */
   IOCTLCMD(SOCKETS_VERSION) = IOCTLCMD(SOCKETS_FIRST),
   IOCTLCMD(SOCKETS_BIND),

/*
    * This used to be for close() on Windows and Mac OS, but is no longer
    * used for the same reason as accept() above.  It is used instead for
    * sending private symbols to the Mac OS driver.
    */
   IOCTLCMD(SOCKETS_SET_SYMBOLS),
   IOCTLCMD(SOCKETS_CONNECT),

/*
    * The next two values are public (vmci_sockets.h) and cannot be changed.
    * That means the number of values above these cannot be changed either
    * unless the base index (specified below) is updated accordingly.
    */
   IOCTLCMD(SOCKETS_GET_AF_VALUE),
   IOCTLCMD(SOCKETS_GET_LOCAL_CID),
   IOCTLCMD(SOCKETS_GET_SOCK_NAME),
   IOCTLCMD(SOCKETS_GET_SOCK_OPT),
   IOCTLCMD(SOCKETS_GET_VM_BY_NAME),
   IOCTLCMD(SOCKETS_IOCTL),
   IOCTLCMD(SOCKETS_LISTEN),
   IOCTLCMD(SOCKETS_RECV),
   IOCTLCMD(SOCKETS_RECV_FROM),
   IOCTLCMD(SOCKETS_SELECT),
   IOCTLCMD(SOCKETS_SEND),
   IOCTLCMD(SOCKETS_SEND_TO),
   IOCTLCMD(SOCKETS_SET_SOCK_OPT),
   IOCTLCMD(SOCKETS_SHUTDOWN),
   IOCTLCMD(SOCKETS_SOCKET),
   IOCTLCMD(SOCKETS_UUID_2_CID), /* 1991 on Linux. */
   /* END VMCI SOCKETS */

/*
    * We reserve a range of 3 ioctls for VMCI Sockets to grow.  We cannot
    * reserve many ioctls here since we are close to overlapping with vmmon
    * ioctls.  Define a meta-ioctl if running out of this binary space.
    */
   // Must be last.
   IOCTLCMD(SOCKETS_LAST) = IOCTLCMD(SOCKETS_UUID_2_CID) + 3, /* 1994 on Linux. */
   /*
    * The VSockets ioctls occupy the block above.  We define a new range of
    * VMCI ioctls to maintain binary compatibility between the user land and
    * the kernel driver.  Careful, vmmon ioctls start from 2001, so this means
    * we can add only 4 new VMCI ioctls.  Define a meta-ioctl if running out of
    * this binary space.
    */

IOCTLCMD(FIRST2),
   IOCTLCMD(SET_NOTIFY) = IOCTLCMD(FIRST2), /* 1995 on Linux. */
   IOCTLCMD(LAST2),
};

#if defined (__APPLE__)
/*
 * The size of this must match the size of VSockIoctlPrivSyms in
 * modules/vsock/common/vsockIoctl.h.
 */
#pragma pack(push, 1)
struct IOCTLCmd_VMCIMacOS_PrivSyms {
   char data[344];
};
#pragma pack(pop)

enum IOCTLCmd_VMCIMacOS {
   IOCTLCMD_I(SOCKETS_SET_SYMBOLS, struct IOCTLCmd_VMCIMacOS_PrivSyms),
   IOCTLCMD_O(SOCKETS_VERSION, unsigned int),
   IOCTLCMD_O(SOCKETS_GET_AF_VALUE, int),
   IOCTLCMD_O(SOCKETS_GET_LOCAL_CID, unsigned int),
};
#endif // __APPLE__

#if defined _WIN32
/*
 * Windows VMCI ioctl definitions.
 */

/* PUBLIC: For VMCISockets user-mode clients that use CreateFile(). */
#define VMCI_INTERFACE_VSOCK_PUBLIC_NAME TEXT("\\\\.\\VMCI")

/* PUBLIC: For VMCISockets user-mode clients that use NtCreateFile(). */
#define VMCI_INTERFACE_VSOCK_PUBLIC_NAME_NT L"\\??\\VMCI"

/* PUBLIC: For the VMX, which uses CreateFile(). */
#define VMCI_INTERFACE_VMX_PUBLIC_NAME TEXT("\\\\.\\VMCIDev\\VMX")

/* PRIVATE NAMES */
#define VMCI_DEVICE_VMCI_LINK_PATH  L"\\DosDevices\\VMCIDev"
#define VMCI_DEVICE_VSOCK_LINK_PATH L"\\DosDevices\\vmci"
#define VMCI_DEVICE_HOST_NAME_PATH  L"\\Device\\VMCIHostDev"
#define VMCI_DEVICE_GUEST_NAME_PATH L"\\Device\\VMCIGuestDev"
/* PRIVATE NAMES */

/* These values cannot be changed since some of the ioctl values are public. */
#define FILE_DEVICE_VMCI      0x8103
#define VMCI_IOCTL_BASE_INDEX 0x801
#define VMCIIOCTL_BUFFERED(name) \
      CTL_CODE(FILE_DEVICE_VMCI, \
	       VMCI_IOCTL_BASE_INDEX + IOCTLCMD_VMCI_ ## name, \
	       METHOD_BUFFERED, \
	       FILE_ANY_ACCESS)
#define VMCIIOCTL_NEITHER(name) \
      CTL_CODE(FILE_DEVICE_VMCI, \
	       VMCI_IOCTL_BASE_INDEX + IOCTLCMD_VMCI_ ## name, \
	       METHOD_NEITHER, \
	       FILE_ANY_ACCESS)

enum IOCTLCmd_VMCIWin32 {
   IOCTLCMD(DEVICE_GET) = IOCTLCMD(LAST2) + 1,
   IOCTLCMD(SOCKETS_SERVICE_GET),
   IOCTLCMD(SOCKETS_STOP),
};

#define IOCTL_VMCI_VERSION VMCIIOCTL_BUFFERED(VERSION)

/* BEGIN VMCI */
#define IOCTL_VMCI_INIT_CONTEXT \
               VMCIIOCTL_BUFFERED(INIT_CONTEXT)
#define IOCTL_VMCI_HYPERCALL \
               VMCIIOCTL_BUFFERED(HYPERCALL)
#define IOCTL_VMCI_CREATE_DATAGRAM_HANDLE  \
               VMCIIOCTL_BUFFERED(CREATE_DATAGRAM_HANDLE)
#define IOCTL_VMCI_DESTROY_DATAGRAM_HANDLE  \
               VMCIIOCTL_BUFFERED(DESTROY_DATAGRAM_HANDLE)
#define IOCTL_VMCI_NOTIFY_RESOURCE    \
               VMCIIOCTL_BUFFERED(NOTIFY_RESOURCE)
#define IOCTL_VMCI_NOTIFICATIONS_RECEIVE    \
               VMCIIOCTL_BUFFERED(NOTIFICATIONS_RECEIVE)
#define IOCTL_VMCI_VERSION2 \
               VMCIIOCTL_BUFFERED(VERSION2)
#define IOCTL_VMCI_QUEUEPAIR_ALLOC  \
               VMCIIOCTL_BUFFERED(QUEUEPAIR_ALLOC)
#define IOCTL_VMCI_QUEUEPAIR_SETVA  \
               VMCIIOCTL_BUFFERED(QUEUEPAIR_SETVA)
#define IOCTL_VMCI_QUEUEPAIR_SETPAGEFILE  \
               VMCIIOCTL_BUFFERED(QUEUEPAIR_SETPAGEFILE)
#define IOCTL_VMCI_QUEUEPAIR_DETACH  \
               VMCIIOCTL_BUFFERED(QUEUEPAIR_DETACH)
#define IOCTL_VMCI_DATAGRAM_SEND \
               VMCIIOCTL_BUFFERED(DATAGRAM_SEND)
#define IOCTL_VMCI_DATAGRAM_RECEIVE \
               VMCIIOCTL_NEITHER(DATAGRAM_RECEIVE)
#define IOCTL_VMCI_DATAGRAM_REQUEST_MAP \
               VMCIIOCTL_BUFFERED(DATAGRAM_REQUEST_MAP)
#define IOCTL_VMCI_DATAGRAM_REMOVE_MAP \
               VMCIIOCTL_BUFFERED(DATAGRAM_REMOVE_MAP)
#define IOCTL_VMCI_CTX_ADD_NOTIFICATION \
               VMCIIOCTL_BUFFERED(CTX_ADD_NOTIFICATION)
#define IOCTL_VMCI_CTX_REMOVE_NOTIFICATION \
               VMCIIOCTL_BUFFERED(CTX_REMOVE_NOTIFICATION)
#define IOCTL_VMCI_CTX_GET_CPT_STATE \
               VMCIIOCTL_BUFFERED(CTX_GET_CPT_STATE)
#define IOCTL_VMCI_CTX_SET_CPT_STATE \
               VMCIIOCTL_BUFFERED(CTX_SET_CPT_STATE)
#define IOCTL_VMCI_GET_CONTEXT_ID    \
               VMCIIOCTL_BUFFERED(GET_CONTEXT_ID)
#define IOCTL_VMCI_DEVICE_GET \
               VMCIIOCTL_BUFFERED(DEVICE_GET)
/* END VMCI */

/* BEGIN VMCI SOCKETS */
#define IOCTL_VMCI_SOCKETS_VERSION \
               VMCIIOCTL_BUFFERED(SOCKETS_VERSION)
#define IOCTL_VMCI_SOCKETS_BIND \
               VMCIIOCTL_BUFFERED(SOCKETS_BIND)
#define IOCTL_VMCI_SOCKETS_CONNECT \
               VMCIIOCTL_BUFFERED(SOCKETS_CONNECT)
#define IOCTL_VMCI_SOCKETS_GET_AF_VALUE \
               VMCIIOCTL_BUFFERED(SOCKETS_GET_AF_VALUE)
#define IOCTL_VMCI_SOCKETS_GET_LOCAL_CID \
               VMCIIOCTL_BUFFERED(SOCKETS_GET_LOCAL_CID)
#define IOCTL_VMCI_SOCKETS_GET_SOCK_NAME \
               VMCIIOCTL_BUFFERED(SOCKETS_GET_SOCK_NAME)
#define IOCTL_VMCI_SOCKETS_GET_SOCK_OPT \
               VMCIIOCTL_BUFFERED(SOCKETS_GET_SOCK_OPT)
#define IOCTL_VMCI_SOCKETS_GET_VM_BY_NAME \
               VMCIIOCTL_BUFFERED(SOCKETS_GET_VM_BY_NAME)
#define IOCTL_VMCI_SOCKETS_IOCTL \
               VMCIIOCTL_BUFFERED(SOCKETS_IOCTL)
#define IOCTL_VMCI_SOCKETS_LISTEN \
               VMCIIOCTL_BUFFERED(SOCKETS_LISTEN)
#define IOCTL_VMCI_SOCKETS_RECV_FROM \
               VMCIIOCTL_BUFFERED(SOCKETS_RECV_FROM)
#define IOCTL_VMCI_SOCKETS_SELECT \
               VMCIIOCTL_BUFFERED(SOCKETS_SELECT)
#define IOCTL_VMCI_SOCKETS_SEND_TO \
               VMCIIOCTL_BUFFERED(SOCKETS_SEND_TO)
#define IOCTL_VMCI_SOCKETS_SET_SOCK_OPT \
               VMCIIOCTL_BUFFERED(SOCKETS_SET_SOCK_OPT)
#define IOCTL_VMCI_SOCKETS_SHUTDOWN \
               VMCIIOCTL_BUFFERED(SOCKETS_SHUTDOWN)
#define IOCTL_VMCI_SOCKETS_SERVICE_GET \
               VMCIIOCTL_BUFFERED(SOCKETS_SERVICE_GET)
#define IOCTL_VMCI_SOCKETS_STOP \
               VMCIIOCTL_NEITHER(SOCKETS_STOP)
/* END VMCI SOCKETS */

#endif // _WIN32

/*
 * VMCI driver initialization. This block can also be used to
 * pass initial group membership etc.
 */
typedef struct VMCIInitBlock {
   VMCIId             cid;
   VMCIPrivilegeFlags flags;
#ifdef _WIN32
   uint64             event; /* Handle for signalling vmci calls on windows. */
#endif // _WIN32
} VMCIInitBlock;

typedef struct VMCISharedMemInfo {
   VMCIHandle handle;
   uint32     size;
   uint32     result;
   VA64       va; /* Currently only used in the guest. */
   char       pageFileName[VMCI_PATH_MAX];
} VMCISharedMemInfo;

typedef struct VMCIQueuePairAllocInfo_VMToVM {
   VMCIHandle handle;
   VMCIId     peer;
   uint32     flags;
   uint64     produceSize;
   uint64     consumeSize;
#if !defined(VMX86_SERVER) && !defined(VMKERNEL)
   VA64       producePageFile; /* User VA. */
   VA64       consumePageFile; /* User VA. */
   uint64     producePageFileSize; /* Size of the file name array. */
   uint64     consumePageFileSize; /* Size of the file name array. */
#else
   PPN64 *    PPNs;
   uint64     numPPNs;
#endif
   int32      result;
   uint32     _pad;
} VMCIQueuePairAllocInfo_VMToVM;

typedef struct VMCIQueuePairAllocInfo {
   VMCIHandle handle;
   VMCIId     peer;
   uint32     flags;
   uint64     produceSize;
   uint64     consumeSize;
#if !defined(VMX86_SERVER) && !defined(VMKERNEL)
   VA64       ppnVA; /* Start VA of queue pair PPNs. */
#else
   PPN64 *    PPNs;
#endif
   uint64     numPPNs;
   int32      result;
   uint32     version;
} VMCIQueuePairAllocInfo;

typedef struct VMCIQueuePairSetVAInfo {
   VMCIHandle handle;
   VA64       va; /* Start VA of queue pair PPNs. */
   uint64     numPPNs;
   uint32     version;
   int32      result;
} VMCIQueuePairSetVAInfo;

/*
 * For backwards compatibility, here is a version of the
 * VMCIQueuePairPageFileInfo before host support end-points was added.
 * Note that the current version of that structure requires VMX to
 * pass down the VA of the mapped file.  Before host support was added
 * there was nothing of the sort.  So, when the driver sees the ioctl
 * with a parameter that is the sizeof
 * VMCIQueuePairPageFileInfo_NoHostQP then it can infer that the version
 * of VMX running can't attach to host end points because it doesn't
 * provide the VA of the mapped files.
 *
 * The Linux driver doesn't get an indication of the size of the
 * structure passed down from user space.  So, to fix a long standing
 * but unfiled bug, the _pad field has been renamed to version.
 * Existing versions of VMX always initialize the PageFileInfo
 * structure so that _pad, er, version is set to 0.
 *
 * A version value of 1 indicates that the size of the structure has
 * been increased to include two UVA's: produceUVA and consumeUVA.
 * These UVA's are of the mmap()'d queue contents backing files.
 *
 * In addition, if when VMX is sending down the
 * VMCIQueuePairPageFileInfo structure it gets an error then it will
 * try again with the _NoHostQP version of the file to see if an older
 * VMCI kernel module is running.
 */
typedef struct VMCIQueuePairPageFileInfo_NoHostQP {
   VMCIHandle handle;
   VA64       producePageFile; /* User VA. */
   VA64       consumePageFile; /* User VA. */
   uint64     producePageFileSize; /* Size of the file name array. */
   uint64     consumePageFileSize; /* Size of the file name array. */
   int32      result;
   uint32     version;         /* Was _pad. Must be 0. */
} VMCIQueuePairPageFileInfo_NoHostQP;

typedef struct VMCIQueuePairPageFileInfo {
   VMCIHandle handle;
#if !defined(VMX86_SERVER) && !defined(VMKERNEL)
   VA64       producePageFile; /* User VA. */
   VA64       consumePageFile; /* User VA. */
   uint64     producePageFileSize; /* Size of the file name array. */
   uint64     consumePageFileSize; /* Size of the file name array. */
#endif
   int32      result;
   uint32     version;   /* Was _pad. */
   VA64       produceVA; /* User VA of the mapped file. */
   VA64       consumeVA; /* User VA of the mapped file. */
} VMCIQueuePairPageFileInfo;

typedef struct VMCIQueuePairDetachInfo {
   VMCIHandle handle;
   int32      result;
   uint32     _pad;
} VMCIQueuePairDetachInfo;

typedef struct VMCIDatagramSendRecvInfo {
   VA64   addr;
   uint32 len;
   int32  result;
} VMCIDatagramSendRecvInfo;

/* Used to add/remove well-known datagram mappings. */
typedef struct VMCIDatagramMapInfo {
   VMCIId      wellKnownID;
   int         result;
} VMCIDatagramMapInfo;

/* Used to add/remove remote context notifications. */
typedef struct VMCINotifyAddRemoveInfo {
   VMCIId      remoteCID;
   int         result;
} VMCINotifyAddRemoveInfo;

/* Used to set/get current context's checkpoint state. */
typedef struct VMCICptBufInfo {
   VA64        cptBuf;
   uint32      cptType;
   uint32      bufSize;
   int32       result;
   uint32      _pad;
} VMCICptBufInfo;

/* Used to pass notify flag's address to the host driver. */
typedef struct VMCISetNotifyInfo {
   VA64        notifyUVA;
   int32       result;
   uint32      _pad;
} VMCISetNotifyInfo;

#define VMCI_NOTIFY_RESOURCE_QUEUE_PAIR 0
#define VMCI_NOTIFY_RESOURCE_DOOR_BELL  1

#define VMCI_NOTIFY_RESOURCE_ACTION_NOTIFY  0
#define VMCI_NOTIFY_RESOURCE_ACTION_CREATE  1
#define VMCI_NOTIFY_RESOURCE_ACTION_DESTROY 2

/*
 * Used to create and destroy doorbells, and generate a notification
 * for a doorbell or queue pair.
 */

typedef struct VMCINotifyResourceInfo {
   VMCIHandle  handle;
   uint16      resource;
   uint16      action;
   int32       result;
} VMCINotifyResourceInfo;

/*
 * Used to recieve pending notifications for doorbells and queue
 * pairs.
 */

typedef struct VMCINotificationReceiveInfo {
   VA64        dbHandleBufUVA;
   uint64      dbHandleBufSize;
   VA64        qpHandleBufUVA;
   uint64      qpHandleBufSize;
   int32       result;
   uint32      _pad;
} VMCINotificationReceiveInfo;

#if defined(_WIN32) && defined(WINNT_DDK)
/*
 * Used on Windows to expose the API calls that are no longer exported.  This
 * is kernel-mode only, and both sides will have the same bitness, so we can
 * use pointers directly.
 */

/* Version 1. */
typedef struct VMCIDeviceGetInfoVer1 {
   VMCI_DeviceReleaseFct *deviceRelease;
   VMCIDatagram_CreateHndFct *dgramCreateHnd;
   VMCIDatagram_CreateHndPrivFct *dgramCreateHndPriv;
   VMCIDatagram_DestroyHndFct *dgramDestroyHnd;
   VMCIDatagram_SendFct *dgramSend;
   VMCI_GetContextIDFct *getContextId;
   VMCI_VersionFct *version;
   VMCIEvent_SubscribeFct *eventSubscribe;
   VMCIEvent_UnsubscribeFct *eventUnsubscribe;
   VMCIQPair_AllocFct *qpairAlloc;
   VMCIQPair_DetachFct *qpairDetach;
   VMCIQPair_GetProduceIndexesFct *qpairGetProduceIndexes;
   VMCIQPair_GetConsumeIndexesFct *qpairGetConsumeIndexes;
   VMCIQPair_ProduceFreeSpaceFct *qpairProduceFreeSpace;
   VMCIQPair_ProduceBufReadyFct *qpairProduceBufReady;
   VMCIQPair_ConsumeFreeSpaceFct *qpairConsumeFreeSpace;
   VMCIQPair_ConsumeBufReadyFct *qpairConsumeBufReady;
   VMCIQPair_EnqueueFct *qpairEnqueue;
   VMCIQPair_DequeueFct *qpairDequeue;
   VMCIQPair_PeekFct *qpairPeek;
   VMCIQPair_EnqueueVFct *qpairEnqueueV;
   VMCIQPair_DequeueVFct *qpairDequeueV;
   VMCIQPair_PeekVFct *qpairPeekV;
   VMCI_ContextID2HostVmIDFct *contextID2HostVmID;
   VMCI_IsContextOwnerFct *isContextOwner;
   VMCIContext_GetPrivFlagsFct *contextGetPrivFlags;
} VMCIDeviceGetInfoVer1;

/* Version 2. */
typedef struct VMCIDeviceGetInfoVer2 {
   VMCIDoorbell_CreateFct *doorbellCreate;
   VMCIDoorbell_DestroyFct *doorbellDestroy;
   VMCIDoorbell_NotifyFct *doorbellNotify;
} VMCIDeviceGetInfoVer2;

typedef struct VMCIDeviceGetInfoHdr {
   /* Requested API version on input, supported version on output. */
   uint32 apiVersion;
   VMCI_DeviceShutdownFn *deviceShutdownCB;
   void *userData;
   void *deviceRegistration;
} VMCIDeviceGetInfoHdr;

/* Combination of all versions. */
typedef struct VMCIDeviceGetInfo {
   VMCIDeviceGetInfoHdr hdr;
   VMCIDeviceGetInfoVer1 ver1;
   VMCIDeviceGetInfoVer2 ver2;
} VMCIDeviceGetInfo;
#endif // _WIN32 && WINNT_DDK

#ifdef __APPLE__
/*
 * Mac OS ioctl definitions.
 *
 * Mac OS defines _IO* macros, and the core kernel code uses the size encoded
 * in the ioctl value to copy the memory back and forth (depending on the
 * direction encoded in the ioctl value) between the user and kernel address
 * spaces.
 * See iocontrolsMacOS.h for details on how this is done. We use sockets only
 * for vmci.
 */

#include <sys/ioccom.h>

enum VMCrossTalkSockOpt {
   VMCI_SO_VERSION = 0,
   VMCI_SO_CONTEXT                  = IOCTL_VMCI_INIT_CONTEXT,
   VMCI_SO_NOTIFY_RESOURCE          = IOCTL_VMCI_NOTIFY_RESOURCE,
   VMCI_SO_NOTIFICATIONS_RECEIVE    = IOCTL_VMCI_NOTIFICATIONS_RECEIVE,
   VMCI_SO_VERSION2                 = IOCTL_VMCI_VERSION2,
   VMCI_SO_QUEUEPAIR_ALLOC          = IOCTL_VMCI_QUEUEPAIR_ALLOC,
   VMCI_SO_QUEUEPAIR_SETVA          = IOCTL_VMCI_QUEUEPAIR_SETVA,
   VMCI_SO_QUEUEPAIR_SETPAGEFILE    = IOCTL_VMCI_QUEUEPAIR_SETPAGEFILE,
   VMCI_SO_QUEUEPAIR_DETACH         = IOCTL_VMCI_QUEUEPAIR_DETACH,
   VMCI_SO_DATAGRAM_SEND            = IOCTL_VMCI_DATAGRAM_SEND,
   VMCI_SO_DATAGRAM_RECEIVE         = IOCTL_VMCI_DATAGRAM_RECEIVE,
   VMCI_SO_DATAGRAM_REQUEST_MAP     = IOCTL_VMCI_DATAGRAM_REQUEST_MAP,
   VMCI_SO_DATAGRAM_REMOVE_MAP      = IOCTL_VMCI_DATAGRAM_REMOVE_MAP,
   VMCI_SO_CTX_ADD_NOTIFICATION     = IOCTL_VMCI_CTX_ADD_NOTIFICATION,
   VMCI_SO_CTX_REMOVE_NOTIFICATION  = IOCTL_VMCI_CTX_REMOVE_NOTIFICATION,
   VMCI_SO_CTX_GET_CPT_STATE        = IOCTL_VMCI_CTX_GET_CPT_STATE,
   VMCI_SO_CTX_SET_CPT_STATE        = IOCTL_VMCI_CTX_SET_CPT_STATE,
   VMCI_SO_GET_CONTEXT_ID           = IOCTL_VMCI_GET_CONTEXT_ID,
   VMCI_SO_USERFD,
};

#define VMCI_MACOS_HOST_DEVICE "com.vmware.kext.vmci"

#endif

/* Clean up helper macros */
#undef IOCTLCMD

#if defined __cplusplus
} // extern "C"
#endif

#endif // ifndef _VMCI_IOCONTROLS_H_
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/kernelStubsSal.h����������������������������������������������������������������0000444�0000000�0000000�00000012205�13432725330�016517� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2015-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * kernelStubsSal.h
 *
 * Contains definitions source annotation language definitions for kernel drivers.
 * This solves two issues:
 * 1. Microsoft changed their annotation language from SAL 1.0 (original one
 *    widely distributed by the Windows team) to their more final SAL 2.0
 *    langauge (championed by the VS team).
 * 2. We want these annotations to do nothing during non-Win32 compiles.
 *
 * A longer term goal is to rationalize this into Bora.
 */
#ifndef __KERNELSTUBSSAL_H__
#define __KERNELSTUBSSAL_H__

#if defined(_WIN32)
#  include <DriverSpecs.h>
#  if !defined(_SAL_VERSION)
#     define _SAL_VERSION 10
#  endif
#endif

#if !defined(_SAL_VERSION) || (defined(_SAL_VERSION) && _SAL_VERSION == 10)
#define _In_
#define _In_opt_
#define _In_reads_bytes_(count)
#define _In_reads_bytes_opt_(count)
#define _In_z_
#define _In_opt_z_
#define _Out_
#define _Out_opt_
#define _Out_writes_bytes_(capcount)
#define _Out_writes_bytes_opt_(capcount)
#define _Out_writes_bytes_to_(cap, count)
#define _Out_writes_bytes_to_opt_(cap, count)
#define _Out_bytecap_post_bytecount_(cap, count)
#define _Out_writes_z_(cap)
#define _Out_writes_opt_z_(cap)
#define _Out_z_cap_(e)
#define _Outptr_result_buffer_(count)
#define _Outptr_result_bytebuffer_(count)
#define _Outptr_result_bytebuffer_maybenull_(count)
#define _Outptr_opt_result_buffer_(count)
#define _Outptr_opt_result_bytebuffer_(count)
#define _Outptr_opt_result_bytebuffer_maybenull_(count)
#define _COM_Outptr_
#define _Inout_
#define _Inout_updates_bytes_(e)
#define _Inout_z_cap_(e)
#define _Post_z_count_(e)
#define _Ret_writes_z_(e)
#define _Ret_writes_maybenull_z_(e)
#define _Ret_maybenull_
#define _Ret_maybenull_z_
#define _Ret_range_(l,h)
#define _Success_(expr)
#define _Check_return_
#define _Must_inspect_result_
#define _Group_(annos)
#define _When_(expr, annos)
#define _Always_(annos)
#define _Printf_format_string_
#define _Use_decl_annotations_
#define _Dispatch_type_(mj)
#define _Function_class_(c)
#define _Requires_lock_held_(cs)
#define _Requires_lock_not_held_(cs)
#define _Acquires_lock_(l)
#define _Releases_lock_(l)
#define _IRQL_requires_max_(i)
#define _IRQL_requires_(i)
#define _IRQL_requires_same_
#define _Analysis_assume_(e)
#define _Pre_notnull_
#define _At_(expr,annos)

#else
// Sal 2.0 path - everything is already defined.
#endif // _SAL_VERSION

// Now define our own annotations
#if !defined(_SAL_VERSION) || (defined(_SAL_VERSION) && _SAL_VERSION == 10)
#define _When_windrv_(annos)
#define _Ret_allocates_malloc_mem_opt_bytecap_(_Size)
#define _Ret_allocates_malloc_mem_opt_bytecount_(_Size)
#define _Ret_allocates_malloc_mem_opt_bytecap_post_bytecount_(_Cap,_Count)
#define _Ret_allocates_malloc_mem_opt_z_bytecount_(_Size)
#define _Ret_allocates_malloc_mem_opt_z_
#define _In_frees_malloc_mem_opt_
#else
#define _When_windrv_(annos)                                               annos
#define _Ret_allocates_malloc_mem_opt_bytecap_(_Cap)                       __drv_allocatesMem("Memory") _Must_inspect_result_ _Ret_opt_bytecap_(_Cap)
#define _Ret_allocates_malloc_mem_opt_bytecount_(_Count)                   __drv_allocatesMem("Memory") _Must_inspect_result_ _Ret_opt_bytecount_(_Count)
#define _Ret_allocates_malloc_mem_opt_bytecap_post_bytecount_(_Cap,_Count) __drv_allocatesMem("Memory") _Must_inspect_result_ _Ret_opt_bytecap_(_Cap) _Ret_opt_bytecount_(_Count)
#define _Ret_allocates_malloc_mem_opt_z_bytecount_(_Count)                 __drv_allocatesMem("Memory") _Must_inspect_result_ _Ret_opt_z_bytecount_(_Count)
#define _Ret_allocates_malloc_mem_opt_z_                                   __drv_allocatesMem("Memory") _Must_inspect_result_ _Ret_opt_z_
#define _In_frees_malloc_mem_opt_                                          __drv_freesMem("Memory") _Pre_maybenull_ _Post_invalid_
#endif // _SAL_VERSION

// Best we can do for reallocate with simple annotations: assume old size was fully initialized.
#define _Ret_reallocates_malloc_mem_opt_newbytecap_oldbytecap_(_NewSize, _OldSize) _Ret_allocates_malloc_mem_opt_bytecap_post_bytecount_(_NewSize, _OldSize <= _NewSize ? _OldSize : _NewSize)
#define _Ret_reallocates_malloc_mem_opt_newbytecap_(_NewSize)                      _Ret_allocates_malloc_mem_opt_z_bytecount_(_NewSize)
#define _In_reallocates_malloc_mem_opt_oldptr_                                     _In_frees_malloc_mem_opt_

#endif // __KERNELSTUBSSAL_H__
�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vm_assert.h���������������������������������������������������������������������0000444�0000000�0000000�00000024303�13432725350�015565� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vm_assert.h --
 *
 *	The basic assertion facility for all VMware code.
 *
 *      For proper use, see bora/doc/assert and
 *      http://vmweb.vmware.com/~mts/WebSite/guide/programming/asserts.html.
 */

#ifndef _VM_ASSERT_H_
#define _VM_ASSERT_H_

#define INCLUDE_ALLOW_USERLEVEL

// XXX not necessary except some places include vm_assert.h improperly
#include "vm_basic_types.h"

#ifdef __cplusplus
extern "C" {
#endif

/*
 * Some bits of vmcore are used in VMKernel code and cannot have
 * the VMKERNEL define due to other header dependencies.
 */
#if defined(VMKERNEL) && !defined(VMKPANIC)
#define VMKPANIC 1
#endif

/*
 * Internal macros, functions, and strings
 *
 * The monitor wants to save space at call sites, so it has specialized
 * functions for each situation.  User level wants to save on implementation
 * so it uses generic functions.
 */

#if !defined VMM || defined MONITOR_APP // {

#if defined (VMKPANIC) 
#include "vmk_assert.h"
#else /* !VMKPANIC */
#define _ASSERT_PANIC(name) \
           Panic(_##name##Fmt "\n", __FILE__, __LINE__)
#define _ASSERT_PANIC_BUG(bug, name) \
           Panic(_##name##Fmt " bugNr=%d\n", __FILE__, __LINE__, bug)
#define _ASSERT_PANIC_NORETURN(name) \
           Panic(_##name##Fmt "\n", __FILE__, __LINE__)
#define _ASSERT_PANIC_BUG_NORETURN(bug, name) \
           Panic(_##name##Fmt " bugNr=%d\n", __FILE__, __LINE__, bug)
#endif /* VMKPANIC */

#endif // }

// These strings don't have newline so that a bug can be tacked on.
#define _AssertPanicFmt            "PANIC %s:%d"
#define _AssertAssertFmt           "ASSERT %s:%d"
#define _AssertVerifyFmt           "VERIFY %s:%d"
#define _AssertNotImplementedFmt   "NOT_IMPLEMENTED %s:%d"
#define _AssertNotReachedFmt       "NOT_REACHED %s:%d"
#define _AssertMemAllocFmt         "MEM_ALLOC %s:%d"
#define _AssertNotTestedFmt        "NOT_TESTED %s:%d"

/*
 * Panic and log functions
 */

void Log(const char *fmt, ...) PRINTF_DECL(1, 2);
void Warning(const char *fmt, ...) PRINTF_DECL(1, 2);
#if defined VMKPANIC
void Panic_SaveRegs(void);

NORETURN void Panic_NoSave(const char *fmt, ...) PRINTF_DECL(1, 2);

#define Panic(fmt...) do { \
   Panic_SaveRegs();       \
   Panic_NoSave(fmt);      \
} while(0)

#else
NORETURN void Panic(const char *fmt, ...) PRINTF_DECL(1, 2);
#endif

void LogThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);
void WarningThrottled(uint32 *count, const char *fmt, ...) PRINTF_DECL(2, 3);

#ifndef ASSERT_IFNOT
   /*
    * PR 271512: When compiling with gcc, catch assignments inside an ASSERT.
    *
    * 'UNLIKELY' is defined with __builtin_expect, which does not warn when
    * passed an assignment (gcc bug 36050). To get around this, we put 'cond'
    * in an 'if' statement and make sure it never gets executed by putting
    * that inside of 'if (0)'. We use gcc's statement expression syntax to
    * make ASSERT an expression because some code uses it that way.
    *
    * Since statement expression syntax is a gcc extension and since it's
    * not clear if this is a problem with other compilers, the ASSERT
    * definition was not changed for them. Using a bare 'cond' with the
    * ternary operator may provide a solution.
    */

#ifdef __GNUC__
      #define ASSERT_IFNOT(cond, panic)                                       \
         ({if (UNLIKELY(!(cond))) { panic; if (0) { if (cond) {;}}} (void)0;})
   #else
      #define ASSERT_IFNOT(cond, panic)                                       \
         (UNLIKELY(!(cond)) ? (panic) : (void)0)
   #endif
#endif

/*
 * Assert, panic, and log macros
 *
 * Some of these are redefined below undef !VMX86_DEBUG.
 * ASSERT() is special cased because of interaction with Windows DDK.
 */

#if defined VMX86_DEBUG
#undef  ASSERT
#define ASSERT(cond) ASSERT_IFNOT(cond, _ASSERT_PANIC(AssertAssert))
#define ASSERT_BUG(bug, cond) \
           ASSERT_IFNOT(cond, _ASSERT_PANIC_BUG(bug, AssertAssert))
#endif

#undef  VERIFY
#define VERIFY(cond) \
           ASSERT_IFNOT(cond, _ASSERT_PANIC_NORETURN(AssertVerify))
#define VERIFY_BUG(bug, cond) \
           ASSERT_IFNOT(cond, _ASSERT_PANIC_BUG_NORETURN(bug, AssertVerify))

#define PANIC()        _ASSERT_PANIC(AssertPanic)
#define PANIC_BUG(bug) _ASSERT_PANIC_BUG(bug, AssertPanic)

#define ASSERT_NOT_IMPLEMENTED(cond) \
           ASSERT_IFNOT(cond, NOT_IMPLEMENTED())

#if defined VMKPANIC || defined VMM
#define NOT_IMPLEMENTED()        _ASSERT_PANIC_NORETURN(AssertNotImplemented)
#else
#define NOT_IMPLEMENTED()        _ASSERT_PANIC(AssertNotImplemented)
#endif

#if defined VMM
#define NOT_IMPLEMENTED_BUG(bug) \
          _ASSERT_PANIC_BUG_NORETURN(bug, AssertNotImplemented)
#else 
#define NOT_IMPLEMENTED_BUG(bug) _ASSERT_PANIC_BUG(bug, AssertNotImplemented)
#endif

#if defined VMKPANIC || defined VMM
#define NOT_REACHED()            _ASSERT_PANIC_NORETURN(AssertNotReached)
#else
#define NOT_REACHED()            _ASSERT_PANIC(AssertNotReached)
#endif

#define ASSERT_MEM_ALLOC(cond) \
           ASSERT_IFNOT(cond, _ASSERT_PANIC(AssertMemAlloc))

#ifdef VMX86_DEVEL
#define NOT_TESTED()       Warning(_AssertNotTestedFmt "\n", __FILE__, __LINE__)
#else
#define NOT_TESTED()       Log(_AssertNotTestedFmt "\n", __FILE__, __LINE__)
#endif

#define ASSERT_NO_INTERRUPTS()  ASSERT(!INTERRUPTS_ENABLED())
#define ASSERT_HAS_INTERRUPTS() ASSERT(INTERRUPTS_ENABLED())

#define ASSERT_NOT_TESTED(cond) (UNLIKELY(!(cond)) ? NOT_TESTED() : (void)0)
#define NOT_TESTED_ONCE()       DO_ONCE(NOT_TESTED())

#define NOT_TESTED_1024()                                               \
   do {                                                                 \
      static uint16 count = 0;                                          \
      if (UNLIKELY(count == 0)) { NOT_TESTED(); }                       \
      count = (count + 1) & 1023;                                       \
   } while (0)

#define LOG_ONCE(_s) DO_ONCE(Log _s)

/*
 * Redefine macros that are only in debug versions
 */

#if !defined VMX86_DEBUG // {

#undef  ASSERT
#define ASSERT(cond)          ((void)0)
#define ASSERT_BUG(bug, cond) ((void)0)

/*
 * Expand NOT_REACHED() as appropriate for each situation.
 *
 * Mainly, we want the compiler to infer the same control-flow
 * information as it would from Panic().  Otherwise, different
 * compilation options will lead to different control-flow-derived
 * errors, causing some make targets to fail while others succeed.
 *
 * VC++ has the __assume() built-in function which we don't trust
 * (see bug 43485); gcc has no such construct; we just panic in
 * userlevel code.  The monitor doesn't want to pay the size penalty
 * (measured at 212 bytes for the release vmm for a minimal infinite
 * loop; panic would cost even more) so it does without and lives
 * with the inconsistency.
 */

#if defined VMKPANIC || defined VMM
#undef  NOT_REACHED
#if defined __GNUC__ && (__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 5)
#define NOT_REACHED() (__builtin_unreachable())
#else
#define NOT_REACHED() ((void)0)
#endif
#else
// keep debug definition
#endif

#undef LOG_UNEXPECTED
#define LOG_UNEXPECTED(bug)     ((void)0)

#undef  ASSERT_NOT_TESTED
#define ASSERT_NOT_TESTED(cond) ((void)0)
#undef  NOT_TESTED
#define NOT_TESTED()            ((void)0)
#undef  NOT_TESTED_ONCE
#define NOT_TESTED_ONCE()       ((void)0)
#undef  NOT_TESTED_1024
#define NOT_TESTED_1024()       ((void)0)

#endif // !VMX86_DEBUG }

/*
 * Compile-time assertions.
 *
 * ASSERT_ON_COMPILE does not use the common
 * switch (0) { case 0: case (e): ; } trick because some compilers (e.g. MSVC)
 * generate code for it.
 *
 * The implementation uses both enum and typedef because the typedef alone is
 * insufficient; gcc allows arrays to be declared with non-constant expressions
 * (even in typedefs, where it makes no sense).
 *
 * NOTE: if GCC ever changes so that it ignores unused types altogether, this
 * assert might not fire!  We explicitly mark it as unused because GCC 4.8+
 * uses -Wunused-local-typedefs as part of -Wall, which means the typedef will
 * generate a warning.
 */

#if defined(_Static_assert) || defined(__cplusplus) ||                         \
    !defined(__GNUC__) || __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 6)
#define ASSERT_ON_COMPILE(e) \
   do { \
      enum { AssertOnCompileMisused = ((e) ? 1 : -1) }; \
      UNUSED_TYPE(typedef char AssertOnCompileFailed[AssertOnCompileMisused]); \
   } while (0)
#else
#define ASSERT_ON_COMPILE(e) \
   do {                      \
      _Static_assert(e, #e); \
   } while (0)
#endif

/*
 * To put an ASSERT_ON_COMPILE() outside a function, wrap it
 * in MY_ASSERTS().  The first parameter must be unique in
 * each .c file where it appears.  For example,
 *
 * MY_ASSERTS(FS3_INT,
 *    ASSERT_ON_COMPILE(sizeof(FS3_DiskLock) == 128);
 *    ASSERT_ON_COMPILE(sizeof(FS3_DiskLockReserved) == DISK_BLOCK_SIZE);
 *    ASSERT_ON_COMPILE(sizeof(FS3_DiskBlock) == DISK_BLOCK_SIZE);
 *    ASSERT_ON_COMPILE(sizeof(Hardware_DMIUUID) == 16);
 * )
 *
 * Caution: ASSERT() within MY_ASSERTS() is silently ignored.
 * The same goes for anything else not evaluated at compile time.
 */

#define MY_ASSERTS(name, assertions) \
   static INLINE void name(void) {   \
      assertions                     \
   }

#ifdef __cplusplus
} /* extern "C" */
#endif

#endif /* ifndef _VM_ASSERT_H_ */
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_sock.h�������������������������������������������������������������������0000444�0000000�0000000�00000006002�13432725347�016066� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2003 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_SOCK_H__
#   define __COMPAT_SOCK_H__

#include <linux/stddef.h> /* for NULL */
#include <net/sock.h>

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
static inline wait_queue_head_t *sk_sleep(struct sock *sk)
{
    return sk->sk_sleep;
}
#endif

/*
 * Prior to 2.6.24, there was no sock network namespace member. In 2.6.26, it
 * was hidden behind accessor functions so that its behavior could vary
 * depending on the value of CONFIG_NET_NS.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
# define compat_sock_net(sk)            sock_net(sk)
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
# define compat_sock_net(sk)            sk->sk_net
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 16)

#ifndef CONFIG_FILTER
# define sk_filter(sk, skb, needlock)    0
#endif

/* Taken from 2.6.16's sock.h and modified for macro. */
# define compat_sk_receive_skb(sk, skb, nested)         \
   ({                                                   \
     int rc = NET_RX_SUCCESS;                           \
                                                        \
     if (sk_filter(sk, skb, 0)) {                       \
        kfree_skb(skb);                                 \
     } else {                                           \
        skb->dev = NULL;                                \
        bh_lock_sock(sk);                               \
        if (!sock_owned_by_user(sk)) {                  \
           rc = (sk)->sk_backlog_rcv(sk, skb);          \
        } else {                                        \
           sk_add_backlog(sk, skb);                     \
        }                                               \
        bh_unlock_sock(sk);                             \
     }                                                  \
                                                        \
     sock_put(sk);                                      \
     rc;                                                \
    })
#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20)
# define compat_sk_receive_skb(sk, skb, nested) sk_receive_skb(sk, skb)
#else
# define compat_sk_receive_skb(sk, skb, nested) sk_receive_skb(sk, skb, nested)
#endif

#endif /* __COMPAT_SOCK_H__ */
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_highmem.h����������������������������������������������������������������0000444�0000000�0000000�00000002423�13432725347�016550� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2012 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_HIGHMEM_H__
#   define __COMPAT_HIGHMEM_H__

#include <linux/highmem.h>

#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
#   define compat_kmap_atomic(_page)   kmap_atomic(_page)
#   define compat_kunmap_atomic(_page) kunmap_atomic(_page)
#else
#   define compat_kmap_atomic(_page)   kmap_atomic((_page), KM_USER0)
#   define compat_kunmap_atomic(_page) kunmap_atomic((_page), KM_USER0)
#endif

#endif /* __COMPAT_HIGHMEM_H__ */
���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_namei.h������������������������������������������������������������������0000444�0000000�0000000�00000003416�13432725347�016226� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_NAMEI_H__
#   define __COMPAT_NAMEI_H__

#include <linux/namei.h>

/*
 * In 2.6.25-rc2, dentry and mount objects were removed from the nameidata
 * struct. They were both replaced with a struct path.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25)
#define compat_vmw_nd_to_dentry(nd) (nd).path.dentry
#else
#define compat_vmw_nd_to_dentry(nd) (nd).dentry
#endif

/* In 2.6.25-rc2, path_release(&nd) was replaced with path_put(&nd.path). */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25)
#define compat_path_release(nd) path_put(&(nd)->path)
#else
#define compat_path_release(nd) path_release(nd)
#endif

/* path_lookup was removed in 2.6.39 merge window VFS merge */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)
#define compat_path_lookup(name, flags, nd)     kern_path(name, flags, &((nd)->path))
#else
#define compat_path_lookup(name, flags, nd)     path_lookup(name, flags, nd)
#endif

#endif /* __COMPAT_NAMEI_H__ */
��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_kernel.h�����������������������������������������������������������������0000444�0000000�0000000�00000002735�13432725347�016420� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2004 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_KERNEL_H__
#   define __COMPAT_KERNEL_H__

#include <asm/unistd.h>
#include <linux/kernel.h>

/*
 * container_of was introduced in 2.5.28 but it's easier to check like this.
 */
#ifndef container_of
#define container_of(ptr, type, member) ({			\
        const typeof( ((type *)0)->member ) *__mptr = (ptr);	\
        (type *)( (char *)__mptr - offsetof(type,member) );})
#endif

/*
 * vsnprintf became available in 2.4.10. For older kernels, just fall back on
 * vsprintf.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 10)
#define vsnprintf(str, size, fmt, args) vsprintf(str, fmt, args)
#endif

#endif /* __COMPAT_KERNEL_H__ */
�����������������������������������vmxnet3-only/shared/circList.h����������������������������������������������������������������������0000444�0000000�0000000�00000025262�13432725346�015350� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 *   circList.h --
 *
 * macros, prototypes and struct definitions for double-linked
 * circular lists.
 */

#ifndef _CIRCLIST_H_
#define _CIRCLIST_H_

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#include "includeCheck.h"

#include "vmware.h"

#if defined(__cplusplus)
extern "C" {
#endif

typedef struct ListItem {
   struct ListItem *prev;
   struct ListItem *next;
} ListItem;

/*
 *----------------------------------------------------------------------
 *
 * CircList_IsEmpty --
 *
 *      A NULL list is an empty list.
 *
 * Result:
 *      TRUE if list is empty, FALSE otherwise.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
CircList_IsEmpty(const ListItem *item)  // IN
{
   return item == NULL;
}

/*
 *----------------------------------------------------------------------
 *
 * CircList_InitItem --
 *
 *      Initialize item as a single-element circular list.
 *
 * Result:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE void
CircList_InitItem(ListItem *item)  // OUT
{
   item->prev = item->next = item;
}

/*
 *----------------------------------------------------------------------
 *
 * CircList_First --
 *
 *      Return first item in the list.
 *
 * Result:
 *      First item.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE ListItem *
CircList_First(ListItem *item)  // IN
{
   return item;
}

/*
 *----------------------------------------------------------------------
 *
 * CircList_Last --
 *
 *      Return last item in the list.
 *
 * Result:
 *      Last item.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE ListItem *
CircList_Last(ListItem *item)
{
   return item->prev;
}

/*
 * CIRC_LIST_CONTAINER - get the struct for this entry (like list_entry)
 * @ptr: the &struct ListItem pointer.
 * @type:   the type of the struct this is embedded in.
 * @member: the name of the list struct within the struct.
 */
#define CIRC_LIST_CONTAINER(ptr, type, member) \
   VMW_CONTAINER_OF(ptr, type, member)
/* 
 * Historical name, left here to reduce churn.
 * TODO: remove, all LIST_CONTAINER uses should be
 * VMW_CONTAINER_OF and stop depending on circList.h
 * to provide the definition.
 */
#define LIST_CONTAINER(ptr, type, member) VMW_CONTAINER_OF(ptr, type, member)

/*
 * LIST_SCAN_FROM scans the list from "from" up until "until".
 * The loop variable p should not be destroyed in the process.
 * "from" is an element in the list where to start scanning.
 * "until" is the element where search should stop.
 * member is the field to use for the search - either "next" or "prev".
 */
#define CIRC_LIST_SCAN_FROM(p, from, until, member)   \
   for (p = (from); (p) != NULL;   \
      (p) = (((p)->member == (until)) ? NULL : (p)->member))

/* scan the entire list (non-destructively) */
#define CIRC_LIST_SCAN(p, l)   \
   CIRC_LIST_SCAN_FROM(p, CircList_First(l), CircList_First(l), next)

/* scan the entire list where loop element may be destroyed */
#define CIRC_LIST_SCAN_SAFE(p, pn, l)   \
   if (!CircList_IsEmpty(l))  \
      for (p = (l), (pn) = CircList_Next(p, l); (p) != NULL;   \
           (p) = (pn), (pn) = CircList_Next(p, l))

/* scan the entire list backwards where loop element may be destroyed */
#define CIRC_LIST_SCAN_BACK_SAFE(p, pn, l)   \
   if (!CircList_IsEmpty(l))  \
      for (p = CircList_Last(l), (pn) = CircList_Prev(p, l); (p) != NULL;   \
           (p) = (pn), (pn) = CircList_Prev(p, l))

/*
 *----------------------------------------------------------------------
 *
 * CircList_Next --
 *
 *      Returns the next member of a doubly linked list, or NULL if last.
 *      Assumes: p is member of the list headed by head.
 *
 * Result:
 *      If head or p is NULL, return NULL. Otherwise,
 *      next list member (or null if last).
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE ListItem *
CircList_Next(ListItem *p,        // IN
              ListItem *head)     // IN
{
   if (head == NULL || p == NULL) {
      return NULL;
   }
   /* both p and head are non-null */
   p = p->next;
   return p == head ? NULL : p;
}

/*
 *----------------------------------------------------------------------
 *
 * CircList_Prev --
 *
 *      Returns the prev member of a doubly linked list, or NULL if first.
 *      Assumes: p is member of the list headed by head.
 *
 * Result:
 *      If head or prev is NULL, return NULL. Otherwise,
 *      prev list member (or null if first).
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE ListItem *
CircList_Prev(ListItem *p,        // IN
              ListItem *head)     // IN
{
   if (head == NULL || p == NULL) {
      return NULL;
   }
   /* both p and head are non-null */
   return p == head ? NULL : p->prev;
}

/*
 *----------------------------------------------------------------------
 *
 * CircList_DeleteItem --
 *
 *      Deletes a member of a doubly linked list, possibly modifies the
 *      list header itself.
 *      Assumes neither p nor headp is null and p is a member of *headp.
 *
 * Result:
 *      None
 *
 * Side effects:
 *      Modifies *headp.
 *
 *----------------------------------------------------------------------
 */

static INLINE void
CircList_DeleteItem(ListItem *p,         // IN
                    ListItem **headp)    // IN/OUT
{
   ListItem *next;

ASSERT(p != NULL);
   ASSERT(headp != NULL);

next = p->next;
   if (p == next) {
      *headp = NULL;
   } else {
      next->prev = p->prev;
      p->prev->next = next;
      if (*headp == p) {
         *headp = next;
      }
   }
}

/*
 *----------------------------------------------------------------------
 *
 * CircList_Queue --
 *
 *      Adds a new member to the back of a doubly linked list (queue)
 *      Assumes neither p nor headp is null and p is not a member of *headp.
 *
 * Result:
 *      None
 *
 * Side effects:
 *      Modifies *headp.
 *
 *----------------------------------------------------------------------
 */

static INLINE void
CircList_Queue(ListItem *p,              // IN
               ListItem **headp)         // IN/OUT
{
   ListItem *head;

head = *headp;
   if (CircList_IsEmpty(head)) {
      CircList_InitItem(p);
      *headp = p;
   } else {
      p->prev = head->prev;
      p->next = head;
      p->prev->next = p;
      head->prev = p;
   }
}

/*
 *----------------------------------------------------------------------
 *
 * CircList_Push --
 *
 *      Adds a new member to the front of a doubly linked list (stack)
 *      Assumes neither p nor headp is null and p is not a member of *headp.
 *
 * Result:
 *      None
 *
 * Side effects:
 *      Modifies *headp.
 *
 *----------------------------------------------------------------------
 */

static INLINE void
CircList_Push(ListItem *p,               // IN
              ListItem **headp)          // IN/OUT
{
   CircList_Queue(p, headp);
   *headp = p;
}

/*
 *----------------------------------------------------------------------
 *
 * CircList_Splice --
 *
 *      Make a single list {l1 l2} from {l1} and {l2} and return it.
 *      It is okay for one or both lists to be NULL.
 *      No checking is done. It is assumed that l1 and l2 are two
 *      distinct lists.
 *
 * Result:
 *      A list { l1 l2 }.
 *
 * Side effects:
 *      Modifies l1 and l2 list pointers.
 *
 *----------------------------------------------------------------------
 */

static INLINE ListItem *
CircList_Splice(ListItem *l1,      // IN
                ListItem *l2)      // IN
{
   ListItem *l1Last, *l2Last;

if (CircList_IsEmpty(l1)) {
      return l2;
   }

if (CircList_IsEmpty(l2)) {
      return l1;
   }

l1Last = l1->prev;   /* last elem of l1 */
   l2Last = l2->prev;   /* last elem of l2 */

/*
    *    l1 -> ... -> l1Last    l2 -> ... l2Last
    */
   l1Last->next = l2;
   l2->prev = l1Last;

l1->prev = l2Last;
   l2Last->next = l1;

return l1;
}

#if 0  /* Presently unused, enable if a use is found */
/*
 *----------------------------------------------------------------------
 *
 * CircList_Split --
 *
 *      Make a list l = {l1 l2} into two separate lists {l1} and {l2}, where:
 *      l = { ... x -> p -> ... } split into:
 *      l1 = { ... -> x }
 *      l2 = { p -> ... }
 *      Assumes neither p nor l is null and p is a member of l.
 *      If p is the first element of l, then l1 will be NULL.
 *
 * Result:
 *      None.
 *
 * Side effects:
 *      Sets *l1p and *l2p to the resulting two lists.
 *      Modifies l's pointers.
 *
 *----------------------------------------------------------------------
 */

static INLINE void
CircList_Split(ListItem *p,         // IN
               ListItem *l,         // IN
               ListItem **l1p,      // OUT
               ListItem **l2p)      // OUT
{
   ListItem *last;

if (p == CircList_First(l)) {   /* first element */
      *l1p = NULL;
      *l2p = l;
      return;
   }

last = l->prev;

*l1p = l;
   p->prev->next = l;
   l->prev = p->prev;

*l2p = p;
   p->prev = last;
   last->next = p;
}
#endif

/*
 *----------------------------------------------------------------------
 *
 * CircList_Size --
 *
 *	Return the number of items in the list.
 *
 * Result:
 *	The number of items in the list.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE int
CircList_Size(ListItem *head)     // IN
{
   ListItem *li;
   int ret = 0;

CIRC_LIST_SCAN(li, head) {
      ret++;
   }
   return ret;
}

#if defined(__cplusplus)
}  // extern "C"
#endif

#endif /* _CIRCLIST_H_ */
����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vmware.h������������������������������������������������������������������������0000444�0000000�0000000�00000003507�13432725350�015066� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2003-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmware.h --
 *
 *	Standard include file for VMware source code.
 */

#ifndef _VMWARE_H_
#define _VMWARE_H_

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_DISTRIBUTE
#include "includeCheck.h"

#include "vm_basic_types.h"
#include "vm_basic_defs.h"
#include "vm_assert.h"

/*
 * Global error codes. Currently used internally, but may be exported
 * to customers one day, like VM_E_XXX in vmcontrol_constants.h
 */

typedef enum VMwareStatus {
   VMWARE_STATUS_SUCCESS,  /* success */
   VMWARE_STATUS_ERROR,    /* generic error */
   VMWARE_STATUS_NOMEM,    /* generic memory allocation error */
   VMWARE_STATUS_INSUFFICIENT_RESOURCES, /* internal or system resource limit exceeded */
   VMWARE_STATUS_INVALID_ARGS  /* invalid arguments */
} VMwareStatus;

#define VMWARE_SUCCESS(s) ((s) == VMWARE_STATUS_SUCCESS)

#endif // ifndef _VMWARE_H_
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_netdevice.h��������������������������������������������������������������0000444�0000000�0000000�00000024324�13432725347�017104� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_NETDEVICE_H__
#   define __COMPAT_NETDEVICE_H__

#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/pci.h>

/*
 * The enet_statistics structure moved from linux/if_ether.h to
 * linux/netdevice.h and is renamed net_device_stats in 2.1.25 --hpreg
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 1, 25)
#   include <linux/if_ether.h>

#   define net_device_stats enet_statistics
#endif

/* The netif_rx_ni() API appeared in 2.4.8 --hpreg */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 8)
#   define netif_rx_ni netif_rx
#endif

/* The device struct was renamed net_device in 2.3.14 --hpreg */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 3, 14)
#   define net_device device
#endif

/*
 * SET_MODULE_OWNER appeared sometime during 2.3.x. It was setting
 * dev->owner = THIS_MODULE until 2.5.70, where netdevice refcounting
 * was completely changed.  SET_MODULE_OWNER was nop for whole
 * 2.6.x series, and finally disappeared in 2.6.24.
 *
 * MOD_xxx_USE_COUNT wrappers are here, as they must be mutually
 * exclusive with SET_MODULE_OWNER call.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 0)
#   define COMPAT_SET_MODULE_OWNER(dev) do {} while (0)
#   define COMPAT_NETDEV_MOD_INC_USE_COUNT MOD_INC_USE_COUNT
#   define COMPAT_NETDEV_MOD_DEC_USE_COUNT MOD_DEC_USE_COUNT
#else
#   if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0)
#      define COMPAT_SET_MODULE_OWNER(dev) SET_MODULE_OWNER(dev)
#   else
#      define COMPAT_SET_MODULE_OWNER(dev) do {} while (0)
#   endif
#   define COMPAT_NETDEV_MOD_INC_USE_COUNT do {} while (0)
#   define COMPAT_NETDEV_MOD_DEC_USE_COUNT do {} while (0)
#endif

/*
 * SET_NETDEV_DEV appeared sometime during 2.5.x, and later was
 * crossported to various 2.4.x kernels (as dummy macro).
 */
#ifdef SET_NETDEV_DEV
#   define COMPAT_SET_NETDEV_DEV(dev, pdev) SET_NETDEV_DEV(dev, pdev)
#else
#   define COMPAT_SET_NETDEV_DEV(dev, pdev) do {} while (0)
#endif

/*
 * Build alloc_etherdev API on the top of init_etherdev.  For 2.0.x kernels
 * we must provide dummy init method, otherwise register_netdev does
 * nothing.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 3)

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 2, 0)
int
vmware_dummy_init(struct net_device *dev)
{
   return 0;
}
#endif

static inline struct net_device*
compat_alloc_etherdev(int priv_size)
{
   struct net_device* dev;
   int size = sizeof *dev + priv_size;

/*
    * The name is dynamically allocated before 2.4.0, but 
    * is an embedded array in later kernels.
    */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 0)
   size += sizeof("ethXXXXXXX");
#endif
   dev = kmalloc(size, GFP_KERNEL);
   if (dev) {
      memset(dev, 0, size);
      if (priv_size) {
         dev->priv = dev + 1;
      }
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 0)
      dev->name = (char *)(dev + 1) + priv_size;
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 2, 0)
      dev->init = vmware_dummy_init;
#endif
      if (init_etherdev(dev, 0) != dev) {
         kfree(dev);
         dev = NULL;
      }
   }
   return dev;
}
#else
#define compat_alloc_etherdev(sz)   alloc_etherdev(sz)
#endif

/*
 * alloc_netdev and free_netdev are there since 2.4.23.  Their use is mandatory
 * since 2.6.24.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 23)
static inline struct net_device *
compat_alloc_netdev(int priv_size,
                    const char *mask,
                    void (*setup)(struct net_device *))
{
   struct net_device *dev;
   int netdev_size = sizeof *dev;
   int alloc_size;

#   if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 0)
   netdev_size += IFNAMSIZ;
#   endif

alloc_size = netdev_size + priv_size;
   dev = kmalloc(alloc_size, GFP_KERNEL);
   if (dev) {
      memset(dev, 0, alloc_size);
      dev->priv = (char*)dev + netdev_size;
      setup(dev);
#   if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 0)
      dev->name = (char*)(dev + 1);
#   endif
      strcpy(dev->name, mask);   
   }
   return dev;
}
#   define compat_free_netdev(dev)     kfree(dev)
#else
#   define compat_alloc_netdev(size, mask, setup) alloc_netdev(size, mask, setup)
#   define compat_free_netdev(dev)                free_netdev(dev)
#endif

/* netdev_priv() appeared in 2.6.3 */
#if  LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 3)
#   define compat_netdev_priv(netdev)   (netdev)->priv
#else
#   define compat_netdev_priv(netdev)   netdev_priv(netdev)
#endif

/*
 * In 3.1 merge window feature maros were removed from mainline,
 * so let's add back ones we care about.
 */
#if !defined(HAVE_NET_DEVICE_OPS) && \
         LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)
#   define HAVE_NET_DEVICE_OPS 1
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 9)
#   define COMPAT_NETDEV_TX_OK    NETDEV_TX_OK
#   define COMPAT_NETDEV_TX_BUSY  NETDEV_TX_BUSY
#else
#   define COMPAT_NETDEV_TX_OK    0
#   define COMPAT_NETDEV_TX_BUSY  1
#endif

/* unregister_netdevice_notifier was not safe prior to 2.6.17 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 17) && \
    !defined(ATOMIC_NOTIFIER_INIT)
/* pre 2.6.17 and not patched */
static inline int compat_unregister_netdevice_notifier(struct notifier_block *nb) {
   int err;

rtnl_lock();
   err = unregister_netdevice_notifier(nb);
   rtnl_unlock();
   return err;
}
#else
/* post 2.6.17 or patched */
#define compat_unregister_netdevice_notifier(_nb) \
        unregister_netdevice_notifier(_nb);
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24) || defined(__VMKLNX__)

#   define compat_netif_napi_add(dev, napi, poll, quota) \
      netif_napi_add(dev, napi, poll, quota)

#   if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 30) || \
       defined VMW_NETIF_SINGLE_NAPI_PARM
#      define compat_napi_complete(dev, napi) napi_complete(napi)
#      define compat_napi_schedule(dev, napi) napi_schedule(napi)
#   else
#      define compat_napi_complete(dev, napi) netif_rx_complete(dev, napi)
#      define compat_napi_schedule(dev, napi) netif_rx_schedule(dev, napi)
#   endif

#   define compat_napi_enable(dev, napi)  napi_enable(napi)
#   define compat_napi_disable(dev, napi) napi_disable(napi)

#else

#   define compat_napi_complete(dev, napi) netif_rx_complete(dev)
#   define compat_napi_schedule(dev, napi) netif_rx_schedule(dev)
#   define compat_napi_enable(dev, napi)   netif_poll_enable(dev)
#   define compat_napi_disable(dev, napi)  netif_poll_disable(dev)

/* RedHat ported GRO to 2.6.18 bringing new napi_struct with it */
#   if defined NETIF_F_GRO
#      define compat_netif_napi_add(netdev, napi, pollcb, quota) \
      do {                        \
         (netdev)->poll = (pollcb);    \
         (netdev)->weight = (quota);\
         (napi)->dev = (netdev); \
      } while (0)

#   else
       struct napi_struct {
          int dummy;
       };
#      define compat_netif_napi_add(dev, napi, pollcb, quota) \
       do {                        \
          (dev)->poll = (pollcb);    \
          (dev)->weight = (quota);\
       } while (0)

#   endif

#endif

#ifdef NETIF_F_TSO6
#  define COMPAT_NETIF_F_TSO (NETIF_F_TSO6 | NETIF_F_TSO)
#else
#  define COMPAT_NETIF_F_TSO (NETIF_F_TSO)
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
#   define compat_netif_tx_lock(dev) netif_tx_lock(dev)
#   define compat_netif_tx_unlock(dev) netif_tx_unlock(dev)
#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 16)
#   define compat_netif_tx_lock(dev) spin_lock(&dev->xmit_lock)
#   define compat_netif_tx_unlock(dev) spin_unlock(&dev->xmit_lock)
#else
/* Vendor backporting (SLES 10) has muddled the tx_lock situation. Pick whichever
 * of the above works for you. */
#   define compat_netif_tx_lock(dev) do {} while (0)
#   define compat_netif_tx_unlock(dev) do {} while (0)
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)
#   define COMPAT_VLAN_GROUP_ARRAY_LEN VLAN_N_VID
#   define compat_flush_scheduled_work(work) cancel_work_sync(work)
#else
#   define COMPAT_VLAN_GROUP_ARRAY_LEN VLAN_GROUP_ARRAY_LEN
#   define compat_flush_scheduled_work(work) flush_scheduled_work()
#endif

/*
 * For kernel versions older than 2.6.29, where pci_msi_enabled is not
 * available, check if
 *	1. CONFIG_PCI_MSI is present
 *	2. kernel version is newer than 2.6.25 (because multiqueue is not
 *	   supporter) in kernels older than that)
 *	3. msi can be enabled. If it fails it means that MSI is not available.
 * When all the above are true, return non-zero so that multiple queues will be
 * allowed in the driver.
 */

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
#   define compat_multiqueue_allowed(dev) pci_msi_enabled()
#else
#   if defined CONFIG_PCI_MSI && LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 25)
static inline int
compat_multiqueue_allowed(struct pci_dev *dev)
{
   int ret;

if (!pci_enable_msi(dev))
      ret = 1;
   else
      ret = 0;

pci_disable_msi(dev);
   return ret;
}

#   else
#      define compat_multiqueue_allowed(dev) (0)
#   endif
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 37)
#   define compat_vlan_get_protocol(skb) vlan_get_protocol(skb)
#else
#   define compat_vlan_get_protocol(skb) (skb->protocol)
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)
typedef netdev_features_t compat_netdev_features_t;
#else
typedef u32 compat_netdev_features_t;
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) || defined(VMW_NETIF_TRANS_UPDATE)
#define compat_netif_trans_update(d) netif_trans_update(d)
#else
#define compat_netif_trans_update(d) do { (d)->trans_start = jiffies; } while (0)
#endif

#endif /* __COMPAT_NETDEVICE_H__ */
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_dcache.h�����������������������������������������������������������������0000444�0000000�0000000�00000004003�13432725347�016335� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2013 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_DCACHE_H__
#   define __COMPAT_DCACHE_H__

#include <linux/dcache.h>

/*
 * per-dentry locking was born in 2.5.62.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 62)
#define compat_lock_dentry(dentry) spin_lock(&dentry->d_lock)
#define compat_unlock_dentry(dentry) spin_unlock(&dentry->d_lock)
#else
#define compat_lock_dentry(dentry) do {} while (0)
#define compat_unlock_dentry(dentry) do {} while (0)
#endif

/*
 * d_alloc_name was born in 2.6.10.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 10)
#define compat_d_alloc_name(parent, s) d_alloc_name(parent, s)
#else
#define compat_d_alloc_name(parent, s)                                        \
({                                                                            \
   struct qstr q;                                                             \
   q.name = s;                                                                \
   q.len = strlen(s);                                                         \
   q.hash = full_name_hash(q.name, q.len);                                    \
   d_alloc(parent, &q);                                                       \
})
#endif

#endif /* __COMPAT_DCACHE_H__ */
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_pci_mapping.h������������������������������������������������������������0000444�0000000�0000000�00000004741�13432725347�017425� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2008 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_PCI_MAPPING_H__
#define __COMPAT_PCI_MAPPING_H__

#include <asm/types.h>
#include <asm/io.h>
#include <linux/pci.h>

#if LINUX_VERSION_CODE < KERNEL_VERSION(2,3,41)
typedef u32 dma_addr_t;

static __inline__ int 
get_order(unsigned long size)
{
   int order;

size = (size - 1) >> (PAGE_SHIFT - 1);
   order = -1;
   do {
      size >>= 1;
      order++;
   } while (size);
   return order;
}

static inline void *
compat_pci_alloc_consistent(struct pci_dev *hwdev, size_t size, dma_addr_t *dma_handle)
{
   void *ptr = (void *)__get_free_pages(GFP_ATOMIC, get_order(size));
   if (ptr) {
      memset(ptr, 0, size);
      *dma_handle = virt_to_phys(ptr);
   }
   return ptr;
}

static inline void
compat_pci_free_consistent(struct pci_dev *hwdev, size_t size, void *vaddr, 
                           dma_addr_t dma_handle)
{
   free_pages((unsigned long)vaddr, get_order(size));
}

static inline dma_addr_t
compat_pci_map_single(struct pci_dev *hwdev, void *ptr, size_t size, int direction)
{
   return virt_to_phys(ptr);
}

static inline void
compat_pci_unmap_single(struct pci_dev *hwdev, dma_addr_t dma_addr,
                        size_t size, int direction)
{
}

#else
#define compat_pci_alloc_consistent(hwdev, size, dma_handle) \
   pci_alloc_consistent(hwdev, size, dma_handle)
#define compat_pci_free_consistent(hwdev, size, vaddr, dma_handle) \
   pci_free_consistent(hwdev, size, vaddr, dma_handle)
#define compat_pci_map_single(hwdev, ptr, size, direction) \
   pci_map_single(hwdev, ptr, size, direction)
#define compat_pci_unmap_single(hwdev, dma_addr, size, direction) \
   pci_unmap_single(hwdev, dma_addr, size, direction)
#endif

#endif
�������������������������������vmxnet3-only/shared/vm_basic_asm_x86.h��������������������������������������������������������������0000444�0000000�0000000�00000042724�13432725350�016721� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vm_basic_asm_x86.h
 *
 *	Basic IA32 asm macros
 */

#ifndef _VM_BASIC_ASM_X86_H_
#define _VM_BASIC_ASM_X86_H_

#define INCLUDE_ALLOW_USERLEVEL

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMCORE
#include "includeCheck.h"

#if defined __cplusplus
extern "C" {
#endif

#ifdef VM_X86_64
/*
 * The gcc inline asm uses the "A" constraint which differs in 32 & 64
 * bit mode.  32 bit means eax and edx, 64 means rax or rdx.
 */
#error "x86-64 not supported"
#endif

/*
 * XTEST
 *     Return TRUE if processor is in transaction region.
 *
 */
#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))
static INLINE Bool
xtest(void)
{
   uint8 al;
   __asm__ __volatile__(".byte 0x0f, 0x01, 0xd6    # xtest \n"
                        "setnz %%al\n"
                        : "=a"(al) : : "cc");
   return al;
}

#endif /* __GNUC__ */

/*
 * FXSAVE/FXRSTOR
 *     save/restore SIMD/MMX fpu state
 *
 * The pointer passed in must be 16-byte aligned.
 *
 * Intel and AMD processors behave differently w.r.t. fxsave/fxrstor. Intel
 * processors unconditionally save the exception pointer state (instruction
 * ptr., data ptr., and error instruction opcode). FXSAVE_ES1 and FXRSTOR_ES1
 * work correctly for Intel processors.
 *
 * AMD processors only save the exception pointer state if ES=1. This leads to a
 * security hole whereby one process/VM can inspect the state of another process
 * VM. The AMD recommended workaround involves clobbering the exception pointer
 * state unconditionally, and this is implemented in FXRSTOR_AMD_ES0. Note that
 * FXSAVE_ES1 will only save the exception pointer state for AMD processors if
 * ES=1.
 *
 * The workaround (FXRSTOR_AMD_ES0) only costs 1 cycle more than just doing an
 * fxrstor, on both AMD Opteron and Intel Core CPUs.
 */
#if defined(__GNUC__)
static INLINE void 
FXSAVE_ES1(void *save)
{
   __asm__ __volatile__ ("fxsave %0\n" : "=m" (*(uint8 *)save) : : "memory");
}

static INLINE void 
FXRSTOR_ES1(const void *load)
{
   __asm__ __volatile__ ("fxrstor %0\n"
                         : : "m" (*(const uint8 *)load) : "memory");
}

static INLINE void 
FXRSTOR_AMD_ES0(const void *load)
{
   uint64 dummy = 0;

/*
 * XSAVE/XRSTOR
 *     save/restore GSSE/SIMD/MMX fpu state
 *
 * The pointer passed in must be 64-byte aligned.
 * See above comment for more information.
 */
#if defined(__GNUC__) && (defined(VMM) || defined(VMKERNEL) || defined(FROBOS))

static INLINE void 
XSAVE_ES1(void *save, uint64 mask)
{
#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
   __asm__ __volatile__ (
        ".byte 0x0f, 0xae, 0x21 \n"
        :
        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#endif
}

static INLINE void 
XSAVEOPT_ES1(void *save, uint64 mask)
{
   __asm__ __volatile__ (
        ".byte 0x0f, 0xae, 0x31 \n"
        :
        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
}

static INLINE void 
XRSTOR_ES1(const void *load, uint64 mask)
{
#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
   __asm__ __volatile__ (
        ".byte 0x0f, 0xae, 0x29 \n"
        :
        : "c" ((const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#endif
}

static INLINE void 
XRSTOR_AMD_ES0(const void *load, uint64 mask)
{
   uint64 dummy = 0;

__asm__ __volatile__ 
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
        "fnclex          \n"     // ES=1. Clear it so fild doesn't trap
        "1:              \n"
        "ffree   %%st(7) \n"     // Clear tag bit - avoid poss. stack overflow
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
        ".byte 0x0f, 0xae, 0x29 \n"
        :
        : "m" (dummy), "c" ((const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
#else
        "xrstor %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
#endif
        : "eax", "memory");
}
#endif /* __GNUC__ */

/*
 *-----------------------------------------------------------------------------
 *
 * Div643232 --
 *
 *    Unsigned integer division:
 *       The dividend is 64-bit wide
 *       The divisor  is 32-bit wide
 *       The quotient is 32-bit wide
 *
 *    Use this function if you are certain that:
 *    o Either the quotient will fit in 32 bits,
 *    o Or your code is ready to handle a #DE exception indicating overflow.
 *    If that is not the case, then use Div643264().
 *
 * Results:
 *    Quotient and remainder
 *
 * Side effects:
 *    None
 *
 *-----------------------------------------------------------------------------
 */

#if defined(__GNUC__)

static INLINE void
Div643232(uint64 dividend,   // IN
          uint32 divisor,    // IN
          uint32 *quotient,  // OUT
          uint32 *remainder) // OUT
{
   __asm__(
      "divl %4"
      : "=a" (*quotient),
        "=d" (*remainder)
      : "0" ((uint32)dividend),
        "1" ((uint32)(dividend >> 32)),
        "rm" (divisor)
      : "cc"
   );
}

#elif defined _MSC_VER

static INLINE void
Div643232(uint64 dividend,   // IN
          uint32 divisor,    // IN
          uint32 *quotient,  // OUT
          uint32 *remainder) // OUT
{
   __asm {
      mov  eax, DWORD PTR [dividend]
      mov  edx, DWORD PTR [dividend+4]
      div  DWORD PTR [divisor]
      mov  edi, DWORD PTR [quotient]
      mov  [edi], eax
      mov  edi, DWORD PTR [remainder]
      mov  [edi], edx
   }
}

#else
#error No compiler defined for Div643232
#endif

#if defined(__GNUC__)
/*
 *-----------------------------------------------------------------------------
 *
 * Div643264 --
 *
 *    Unsigned integer division:
 *       The dividend is 64-bit wide
 *       The divisor  is 32-bit wide
 *       The quotient is 64-bit wide
 *
 * Results:
 *    Quotient and remainder
 *
 * Side effects:
 *    None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Div643264(uint64 dividend,   // IN
          uint32 divisor,    // IN
          uint64 *quotient,  // OUT
          uint32 *remainder) // OUT
{
   uint32 hQuotient;
   uint32 lQuotient;

__asm__(
      "divl %5"        "\n\t"
      "movl %%eax, %0" "\n\t"
      "movl %4, %%eax" "\n\t"
      "divl %5"
      : "=&rm" (hQuotient),
        "=a" (lQuotient),
        "=d" (*remainder)
      : "1" ((uint32)(dividend >> 32)),
        "g" ((uint32)dividend),
        "rm" (divisor),
        "2" (0)
      : "cc"
   );
   *quotient = (uint64)hQuotient << 32 | lQuotient;
}
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * Mul64x3264 --
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
 * 
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
 *
 * Result:
 *       Unsigned 64-bit integer product.
 *
 *-----------------------------------------------------------------------------
 */

#if defined(__GNUC__) && \
   (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4)) && \
   !defined(MUL64_NO_ASM)

static INLINE uint64
Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
{
   uint64 result;
   uint32 tmp1, tmp2;
   // ASSERT(shift >= 0 && shift < 64);
  
   __asm__("mov   %%eax, %2\n\t"      // Save lo(multiplicand)
           "mov   %%edx, %%eax\n\t"   // Get hi(multiplicand)
           "mull  %4\n\t"             // p2 = hi(multiplicand) * multiplier
           "xchg  %%eax, %2\n\t"      // Save lo(p2), get lo(multiplicand)
           "mov   %%edx, %1\n\t"      // Save hi(p2)
           "mull  %4\n\t"             // p1 = lo(multiplicand) * multiplier
           "addl  %2, %%edx\n\t"      // hi(p1) += lo(p2)
           "adcl  $0, %1\n\t"         // hi(p2) += carry from previous step
           "cmpl  $32, %%ecx\n\t"     // shift < 32?
           "jl    2f\n\t"             // Go if so
           "shll  $1, %%eax\n\t"      // Save lo(p1) bit 31 in CF in case shift=32
           "mov   %%edx, %%eax\n\t"   // result = hi(p2):hi(p1) >> (shift & 31)
           "mov   %1, %%edx\n\t"
           "shrdl %%edx, %%eax\n\t"
           "mov   $0, %2\n\t"
           "adcl  $0, %2\n\t"         // Get highest order bit shifted out, from CF
           "shrl  %%cl, %%edx\n\t"
           "jmp   3f\n"
        "2:\n\t"
           "xor   %2, %2\n\t"
           "shrdl %%edx, %%eax\n\t"   // result = hi(p2):hi(p1):lo(p1) >> shift
           "adcl  $0, %2\n\t"         // Get highest order bit shifted out, from CF
           "shrdl %1, %%edx\n"
        "3:\n\t"
           "addl  %2, %%eax\n\t"      // result += highest order bit shifted out
           "adcl  $0, %%edx"
           : "=A" (result), "=&r" (tmp1), "=&r" (tmp2)
           : "0" (multiplicand), "rm" (multiplier), "c" (shift)
           : "cc");
   return result;
}

#elif defined _MSC_VER
#pragma warning(disable: 4035)

static INLINE uint64
Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
{
   // ASSERT(shift >= 0 && shift < 64);

__asm {
      mov  eax, DWORD PTR [multiplicand+4]  // Get hi(multiplicand)
      mul  DWORD PTR [multiplier]           // p2 = hi(multiplicand) * multiplier
      mov  ecx, eax                         // Save lo(p2)
      mov  ebx, edx                         // Save hi(p2)
      mov  eax, DWORD PTR [multiplicand]    // Get lo(multiplicand)
      mul  DWORD PTR [multiplier+0]         // p1 = lo(multiplicand) * multiplier
      add  edx, ecx                         // hi(p1) += lo(p2)
      adc  ebx, 0                           // hi(p2) += carry from previous step
      mov  ecx, DWORD PTR [shift]           // Get shift
      cmp  ecx, 32                          // shift < 32?
      jl   SHORT l2                         // Go if so
      shl  eax, 1                           // Save lo(p1) bit 31 in CF in case shift=32
      mov  eax, edx                         // result = hi(p2):hi(p1) >> (shift & 31)
      mov  edx, ebx
      shrd eax, edx, cl
      mov  esi, 0
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shr  edx, cl
      jmp  SHORT l3
   l2:
      xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) >> shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
   l3:
      add  eax, esi                         // result += highest order bit shifted out
      adc  edx, 0
   }
   // return with result in edx:eax
}

#pragma warning(default: 4035)
#else
#define MUL64_NO_ASM 1
#include "mul64.h"
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * Muls64x32s64 --
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
 * 
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
 *
 * Result:
 *       Signed 64-bit integer product.
 *
 *-----------------------------------------------------------------------------
 */

#if defined(__GNUC__) && \
   (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4)) && \
   !defined(MUL64_NO_ASM)

static INLINE int64
Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
{
   int64 result;
   uint32 tmp1, tmp2;
   // ASSERT(shift >= 0 && shift < 64);

__asm__("mov   %%eax, %2\n\t"      // Save lo(multiplicand)
           "mov   %%edx, %%eax\n\t"   // Get hi(multiplicand)
           "test  %%eax, %%eax\n\t"   // Check sign of multiplicand
           "jl    0f\n\t"             // Go if negative
           "mull  %4\n\t"             // p2 = hi(multiplicand) * multiplier
           "jmp   1f\n"
        "0:\n\t"
           "mull  %4\n\t"             // p2 = hi(multiplicand) * multiplier
           "sub   %4, %%edx\n"        // hi(p2) += -1 * multiplier
        "1:\n\t"
           "xchg  %%eax, %2\n\t"      // Save lo(p2), get lo(multiplicand)
           "mov   %%edx, %1\n\t"      // Save hi(p2)
           "mull  %4\n\t"             // p1 = lo(multiplicand) * multiplier
           "addl  %2, %%edx\n\t"      // hi(p1) += lo(p2)
           "adcl  $0, %1\n\t"         // hi(p2) += carry from previous step
           "cmpl  $32, %%ecx\n\t"     // shift < 32?
           "jl    2f\n\t"             // Go if so
           "shll  $1, %%eax\n\t"      // Save lo(p1) bit 31 in CF in case shift=32
           "mov   %%edx, %%eax\n\t"   // result = hi(p2):hi(p1) >> (shift & 31)
           "mov   %1, %%edx\n\t"
           "shrdl %%edx, %%eax\n\t"
           "mov   $0, %2\n\t"
           "adcl  $0, %2\n\t"         // Get highest order bit shifted out from CF
           "sarl  %%cl, %%edx\n\t"
           "jmp   3f\n"
        "2:\n\t"
           "xor   %2, %2\n\t"
           "shrdl %%edx, %%eax\n\t"   // result = hi(p2):hi(p1):lo(p1) >> shift
           "adcl  $0, %2\n\t"         // Get highest order bit shifted out from CF
           "shrdl %1, %%edx\n"
        "3:\n\t"
           "addl  %2, %%eax\n\t"      // result += highest order bit shifted out
           "adcl  $0, %%edx"
           : "=A" (result), "=&r" (tmp1), "=&rm" (tmp2)
           : "0" (multiplicand), "rm" (multiplier), "c" (shift)
           : "cc");
   return result;
}

#elif defined(_MSC_VER)
#pragma warning(disable: 4035)

static INLINE int64
Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
{
   //ASSERT(shift >= 0 && shift < 64);
  
   __asm {
      mov  eax, DWORD PTR [multiplicand+4]  // Get hi(multiplicand)
      test eax, eax                         // Check sign of multiplicand
      jl   SHORT l0                         // Go if negative
      mul  DWORD PTR [multiplier]           // p2 = hi(multiplicand) * multiplier
      jmp  SHORT l1
   l0:
      mul  DWORD PTR [multiplier]           // p2 = hi(multiplicand) * multiplier
      sub  edx, DWORD PTR [multiplier]      // hi(p2) += -1 * multiplier
   l1:
      mov  ecx, eax                         // Save lo(p2)
      mov  ebx, edx                         // Save hi(p2)
      mov  eax, DWORD PTR [multiplicand]    // Get lo(multiplicand)
      mul  DWORD PTR [multiplier]           // p1 = lo(multiplicand) * multiplier
      add  edx, ecx                         // hi(p1) += lo(p2)
      adc  ebx, 0                           // hi(p2) += carry from previous step
      mov  ecx, DWORD PTR [shift]           // Get shift
      cmp  ecx, 32                          // shift < 32?
      jl   SHORT l2                         // Go if so
      shl  eax, 1                           // Save lo(p1) bit 31 in CF in case shift=32
      mov  eax, edx                         // result = hi(p2):hi(p1) >> (shift & 31)
      mov  edx, ebx
      shrd eax, edx, cl
      mov  esi, 0
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      sar  edx, cl
      jmp  SHORT l3
   l2:
      xor  esi, esi
      shrd eax, edx, cl                     // result = hi(p2):hi(p1):lo(p1) << shift
      adc  esi, 0                           // Get highest order bit shifted out, from CF
      shrd edx, ebx, cl
   l3:
      add  eax, esi                         // result += highest order bit shifted out
      adc  edx, 0
   }
   // return with result in edx:eax
}

#pragma warning(default: 4035)
#endif

#if defined __cplusplus
} // extern "C"
#endif

#endif // _VM_BASIC_ASM_X86_H_
��������������������������������������������vmxnet3-only/shared/compat_module.h�����������������������������������������������������������������0000444�0000000�0000000�00000005127�13432725347�016423� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * compat_module.h --
 */

#ifndef __COMPAT_MODULE_H__
#   define __COMPAT_MODULE_H__

#include <linux/module.h>

/*
 * Modules wishing to use the GPL license are required to include a
 * MODULE_LICENSE definition in their module source as of 2.4.10.
 */
#ifndef MODULE_LICENSE
#define MODULE_LICENSE(license)
#endif

/*
 * To make use of our own home-brewed MODULE_INFO, we need macros to
 * concatenate two expressions to "__mod_", and and to convert an
 * expression into a string. I'm sure we've got these in our codebase,
 * but I'd rather not introduce such a dependency in a compat header.
 */
#ifndef __module_cat
#define __module_cat_1(a, b) __mod_ ## a ## b
#define __module_cat(a, b) __module_cat_1(a, b)
#endif

#ifndef __stringify
#define __stringify_1(x) #x
#define __stringify(x) __stringify_1(x)
#endif

/*
 * MODULE_INFO was born in 2.5.69.
 */
#ifndef MODULE_INFO
#define MODULE_INFO(tag, info)                                                \
static const char __module_cat(tag, __LINE__)[]                               \
  __attribute__((section(".modinfo"), unused)) = __stringify(tag) "=" info
#endif

/*
 * MODULE_VERSION was born in 2.6.4. The earlier form appends a long "\0xxx"
 * string to the module's version, but that was removed in 2.6.10, so we'll
 * ignore it in our wrapper.
 */
#ifndef MODULE_VERSION
#define MODULE_VERSION(_version) MODULE_INFO(version, _version)
#endif

/*
 * Linux kernel < 2.6.31 takes 'int' for 'bool' module parameters.
 * Linux kernel >= 3.3.0 takes 'bool' for 'bool' module parameters.
 * Kernels between the two take either.  So flip switch at 3.0.0.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)
   typedef bool compat_mod_param_bool;
#else
   typedef int compat_mod_param_bool;
#endif

#endif /* __COMPAT_MODULE_H__ */
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vm_basic_asm_x86_64.h�����������������������������������������������������������0000444�0000000�0000000�00000042355�13432725350�017232� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vm_basic_asm_x86_64.h
 *
 *	Basic x86_64 asm macros.
 */

#ifndef _VM_BASIC_ASM_X86_64_H_
#define _VM_BASIC_ASM_X86_64_H_

#define INCLUDE_ALLOW_USERLEVEL

#ifndef VM_X86_64
#error "This file is x86-64 only!"
#endif

#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)

#ifdef __cplusplus
extern "C" {
#endif
uint64 _umul128(uint64 multiplier, uint64 multiplicand,
                uint64 *highProduct);
int64 _mul128(int64 multiplier, int64 multiplicand,
              int64 *highProduct);
uint64 __shiftright128(uint64 lowPart, uint64 highPart, uint8 shift);
#ifdef __cplusplus
}
#endif

#pragma intrinsic(_umul128, _mul128, __shiftright128)

#endif // _MSC_VER

#if defined(__GNUC__)
/*
 * GET_CURRENT_PC
 *
 * Returns the current program counter (i.e. instruction pointer i.e. rip
 * register on x86_64). In the example below:
 *
 *   foo.c
 *   L123: Foo(GET_CURRENT_PC())
 *
 * the return value from GET_CURRENT_PC will point a debugger to L123.
 */
#define GET_CURRENT_PC() ({                                           \
      void *__rip;                                                    \
      asm("lea 0(%%rip), %0;\n\t"                                     \
         : "=r" (__rip));                                             \
      __rip;                                                          \
})

/*
 * GET_CURRENT_LOCATION
 *
 * Updates the arguments with the values of the %rip, %rbp, and %rsp
 * registers at the current code location where the macro is invoked,
 * and the return address.
 */
#define GET_CURRENT_LOCATION(rip, rbp, rsp, retAddr)  do {         \
      asm("lea 0(%%rip), %0\n"                                     \
          "mov %%rbp, %1\n"                                        \
          "mov %%rsp, %2\n"                                        \
          : "=r" (rip), "=r" (rbp), "=r" (rsp));                   \
      retAddr = (uint64) GetReturnAddress();                       \
   } while (0)
#endif

static INLINE void 
FXSAVE_ES1(void *save)
{
   __asm__ __volatile__ ("fxsaveq %0  \n" : "=m" (*(uint8 *)save) : : "memory");
}

static INLINE void 
FXSAVE_COMPAT_ES1(void *save)
{
   __asm__ __volatile__ ("fxsave %0  \n" : "=m" (*(uint8 *)save) : : "memory");
}

static INLINE void 
FXRSTOR_ES1(const void *load)
{
   __asm__ __volatile__ ("fxrstorq %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
}

static INLINE void 
FXRSTOR_COMPAT_ES1(const void *load)
{
   __asm__ __volatile__ ("fxrstor %0 \n"
                         : : "m" (*(const uint8 *)load) : "memory");
}

static INLINE void 
FXRSTOR_AMD_ES0(const void *load)
{
   uint64 dummy = 0;

#endif /* __GNUC__ */

static INLINE void 
XSAVE_ES1(void *save, uint64 mask)
{
#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
   __asm__ __volatile__ (
        ".byte 0x48, 0x0f, 0xae, 0x21 \n"
        :
        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#else
   __asm__ __volatile__ (
        "xsaveq %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#endif
}

static INLINE void 
XSAVE_COMPAT_ES1(void *save, uint64 mask)
{
#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
   __asm__ __volatile__ (
        ".byte 0x0f, 0xae, 0x21 \n"
        :
        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#else
   __asm__ __volatile__ (
        "xsave %0 \n"
        : "=m" (*(uint8 *)save)
        : "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#endif
}

static INLINE void 
XSAVEOPT_ES1(void *save, uint64 mask)
{
   __asm__ __volatile__ (
        ".byte 0x48, 0x0f, 0xae, 0x31 \n"
        :
        : "c" ((uint8 *)save), "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
}

static INLINE void 
XRSTOR_ES1(const void *load, uint64 mask)
{
#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
   __asm__ __volatile__ (
        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
        :
        : "c" ((const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#else
   __asm__ __volatile__ (
        "xrstorq %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#endif
}

static INLINE void 
XRSTOR_COMPAT_ES1(const void *load, uint64 mask)
{
#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
   __asm__ __volatile__ (
        ".byte 0x0f, 0xae, 0x29 \n"
        :
        : "c" ((const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#else
   __asm__ __volatile__ (
        "xrstor %0 \n"
        :
        : "m" (*(const uint8 *)load),
          "a" ((uint32)mask), "d" ((uint32)(mask >> 32))
        : "memory");
#endif
}

static INLINE void 
XRSTOR_AMD_ES0(const void *load, uint64 mask)
{
   uint64 dummy = 0;

__asm__ __volatile__ 
       ("fnstsw  %%ax    \n"     // Grab x87 ES bit
        "bt      $7,%%ax \n"     // Test ES bit
        "jnc     1f      \n"     // Jump if ES=0
        "fnclex          \n"     // ES=1. Clear it so fild doesn't trap
        "1:              \n"
        "ffree   %%st(7) \n"     // Clear tag bit - avoid poss. stack overflow
        "fildl   %0      \n"     // Dummy Load from "safe address" changes all
                                 // x87 exception pointers.
        "mov %%ebx, %%eax \n"
#if __GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ == 1
        ".byte 0x48, 0x0f, 0xae, 0x29 \n"
        :
        : "m" (dummy), "c" ((const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
#else
        "xrstorq %1 \n"
        :
        : "m" (dummy), "m" (*(const uint8 *)load),
          "b" ((uint32)mask), "d" ((uint32)(mask >> 32))
#endif
        : "eax", "memory");
}

#endif /* __GNUC__ */

/*
 *-----------------------------------------------------------------------------
 *
 * Mul64x6464 --
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
 * 
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
 *
 * Result:
 *       Unsigned 64-bit integer product.
 *
 *-----------------------------------------------------------------------------
 */

#if defined(__GNUC__) && !defined(MUL64_NO_ASM)

static INLINE uint64
Mul64x6464(uint64 multiplicand,
           uint64 multiplier,
           uint32 shift)
{
   /*
    * Implementation:
    *    Multiply 64x64 bits to yield a full 128-bit product.
    *    Clear the carry bit (needed for the shift == 0 case).
    *    Shift result in RDX:RAX right by "shift".
    *    Add the carry bit.  (If shift > 0, this is the highest order bit
    *      that was discarded by the shift; else it is 0.)
    *    Return the low-order 64 bits of the above.
    *
    */
   uint64 result, dummy;

__asm__("mulq    %3           \n\t"
           "clc                  \n\t"
           "shrdq   %b4, %1, %0  \n\t"
           "adc     $0, %0       \n\t"
           : "=a" (result),
             "=d" (dummy)
           : "0"  (multiplier),
             "rm" (multiplicand),
             "c"  (shift)
           : "cc");
   return result;
}

#elif defined(_MSC_VER) && !defined(MUL64_NO_ASM)

static INLINE uint64
Mul64x6464(uint64 multiplicand,
           uint64 multiplier,
           uint32 shift)
{
   /*
    * Unfortunately, MSVC intrinsics don't give us access to the carry
    * flag after a 128-bit shift, so the implementation is more
    * awkward:
    *    Multiply 64x64 bits to yield a full 128-bit product.
    *    Shift result right by "shift".
    *    If shift != 0, extract and add in highest order bit that was
    *      discarded by the shift.
    *    Return the low-order 64 bits of the above.
    */
   uint64 tmplo, tmphi;
   tmplo = _umul128(multiplicand, multiplier, &tmphi);
   if (shift == 0) {
      return tmplo;
   } else {
      return __shiftright128(tmplo, tmphi, (uint8) shift) +
         ((tmplo >> (shift - 1)) & 1);
   }
}

#else
#define MUL64_NO_ASM 1
#include "mul64.h"
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * Muls64x64s64 --
 *
 *    Signed integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
 * 
 *       Signed 64-bit integer multiplicand.
 *       Unsigned 64-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
 *
 * Result:
 *       Signed 64-bit integer product.
 *
 *-----------------------------------------------------------------------------
 */

#if defined(__GNUC__) && !defined(MUL64_NO_ASM)

static inline int64
Muls64x64s64(int64 multiplicand,
             int64 multiplier,
             uint32 shift)
{
   int64 result, dummy;

/* Implementation:
    *    Multiply 64x64 bits to yield a full 128-bit product.
    *    Clear the carry bit (needed for the shift == 0 case).
    *    Shift result in RDX:RAX right by "shift".
    *    Add the carry bit.  (If shift > 0, this is the highest order bit
    *      that was discarded by the shift; else it is 0.)
    *    Return the low-order 64 bits of the above.
    *
    *    Note: using the unsigned shrd instruction is correct because
    *    shift < 64 and we return only the low 64 bits of the shifted
    *    result.
    */
   __asm__("imulq   %3           \n\t"
           "clc                  \n\t"
           "shrdq   %b4, %1, %0  \n\t"
           "adc     $0, %0       \n\t"
           : "=a" (result),
             "=d" (dummy)
           : "0"  (multiplier),
             "rm" (multiplicand),
             "c"  (shift)
           : "cc");
   return result;
}

#elif defined(_MSC_VER) && !defined(MUL64_NO_ASM)

static INLINE int64
Muls64x64s64(int64 multiplicand,
             int64 multiplier,
             uint32 shift)
{
   /*
    * Unfortunately, MSVC intrinsics don't give us access to the carry
    * flag after a 128-bit shift, so the implementation is more
    * awkward:
    *    Multiply 64x64 bits to yield a full 128-bit product.
    *    Shift result right by "shift".
    *    If shift != 0, extract and add in highest order bit that was
    *      discarded by the shift.
    *    Return the low-order 64 bits of the above.
    *
    * Note: using an unsigned shift is correct because shift < 64 and
    * we return only the low 64 bits of the shifted result.
    */
   int64 tmplo, tmphi;
   tmplo = _mul128(multiplicand, multiplier, &tmphi);
   if (shift == 0) {
      return tmplo;
   } else {
      return __shiftright128(tmplo, tmphi, (uint8) shift) +
         ((tmplo >> (shift - 1)) & 1);
   }
}

#endif

#ifndef MUL64_NO_ASM
/*
 *-----------------------------------------------------------------------------
 *
 * Mul64x3264 --
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
 * 
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
 *
 * Result:
 *       Unsigned 64-bit integer product.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint64
Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
{
   return Mul64x6464(multiplicand, multiplier, shift);
}

static INLINE int64
Muls64x32s64(int64 multiplicand, uint32 multiplier, uint32 shift)
{
   return Muls64x64s64(multiplicand, multiplier, shift);
}
#endif

#if defined(__GNUC__)

static INLINE void *
uint64set(void *dst, uint64 val, uint64 count)
{
   int dummy0;
   int dummy1;
   __asm__ __volatile__("\t"
                        "cld"            "\n\t"
                        "rep ; stosq"    "\n"
                        : "=c" (dummy0), "=D" (dummy1)
                        : "0" (count), "1" (dst), "a" (val)
                        : "memory", "cc");
   return dst;
}

#endif

/*
 *-----------------------------------------------------------------------------
 *
 * Div643232 --
 *
 *    Unsigned integer division:
 *       The dividend is 64-bit wide
 *       The divisor  is 32-bit wide
 *       The quotient is 32-bit wide
 *
 *    Use this function if you are certain that the quotient will fit in 32 bits,
 *    If that is not the case, a #DE exception was generated in 32-bit version,
 *    but not in this 64-bit version. So please be careful.
 *
 * Results:
 *    Quotient and remainder
 *
 * Side effects:
 *    None
 *
 *-----------------------------------------------------------------------------
 */

#if defined(__GNUC__) || defined(_MSC_VER)

static INLINE void
Div643232(uint64 dividend,   // IN
          uint32 divisor,    // IN
          uint32 *quotient,  // OUT
          uint32 *remainder) // OUT
{
   *quotient = (uint32)(dividend / divisor);
   *remainder = (uint32)(dividend % divisor);
}

#endif

/*
 *-----------------------------------------------------------------------------
 *
 * Div643264 --
 *
 *    Unsigned integer division:
 *       The dividend is 64-bit wide
 *       The divisor  is 32-bit wide
 *       The quotient is 64-bit wide
 *
 * Results:
 *    Quotient and remainder
 *
 * Side effects:
 *    None
 *
 *-----------------------------------------------------------------------------
 */

#if defined(__GNUC__)

static INLINE void
Div643264(uint64 dividend,   // IN
          uint32 divisor,    // IN
          uint64 *quotient,  // OUT
          uint32 *remainder) // OUT
{
   *quotient = dividend / divisor;
   *remainder = dividend % divisor;
}

#endif

#endif // _VM_BASIC_ASM_X86_64_H_
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_timer.h������������������������������������������������������������������0000444�0000000�0000000�00000006551�13432725347�016260� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_TIMER_H__
#   define __COMPAT_TIMER_H__

/*
 * The del_timer_sync() API appeared in 2.3.43
 * It became reliable in 2.4.0-test3
 *
 *   --hpreg
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 0)
#   define compat_del_timer_sync(timer) del_timer_sync(timer)
#else
#   if LINUX_VERSION_CODE < KERNEL_VERSION(2, 3, 43)
       /* 2.3.43 removed asm/softirq.h's reference to bh_base. */
#      include <linux/interrupt.h>
#   endif
#   include <asm/softirq.h>

static inline int
compat_del_timer_sync(struct timer_list *timer) // IN
{
   int wasPending;

start_bh_atomic();
   wasPending = del_timer(timer);
   end_bh_atomic();

return wasPending;
}
#endif

/*
 * The msleep_interruptible() API appeared in 2.6.9.
 * It is based on the msleep() API, which appeared in 2.4.29.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 9)
#   include <linux/delay.h>
#   define compat_msleep_interruptible(msecs) msleep_interruptible(msecs)
#   define compat_msleep(msecs) msleep(msecs)
#else
#   include <linux/sched.h>
/* 
 * msecs_to_jiffies appeared in 2.6.7.  For earlier kernels,
 * fall back to slow-case code (we don't use this operation
 * enough to need the performance).
 */
#   if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 7)
#      define msecs_to_jiffies(msecs) (((msecs) * HZ + 999) / 1000)
#   endif
/*
 * set_current_state appeared in 2.2.18.
 */
#   if LINUX_VERSION_CODE < KERNEL_VERSION(2, 2, 18)
#      define set_current_state(a) do { current->state = (a); } while(0)
#   endif

static inline void
compat_msleep_interruptible(unsigned long msecs) // IN
{
   set_current_state(TASK_INTERRUPTIBLE);
   schedule_timeout(msecs_to_jiffies(msecs) + 1);
}

static inline void
compat_msleep(unsigned long msecs) // IN
{
   set_current_state(TASK_UNINTERRUPTIBLE);
   schedule_timeout(msecs_to_jiffies(msecs) + 1);
}
#endif

/*
 * There is init_timer_deferrable() since 2.6.22.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
#   define compat_init_timer_deferrable(timer) init_timer_deferrable(timer)
#else
#   define compat_init_timer_deferrable(timer) init_timer(timer)
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
static inline void compat_setup_timer(struct timer_list * timer,
                                      void (*function)(unsigned long),
                                      unsigned long data)
{
   timer->function = function;
   timer->data = data;
   init_timer(timer);
}
#else
#   define compat_setup_timer(timer, function, data) \
       setup_timer(timer, function, data)
#endif

#endif /* __COMPAT_TIMER_H__ */
�������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/kernelStubsLinux.c��������������������������������������������������������������0000444�0000000�0000000�00000024601�13432725330�017075� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006-2014 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * kernelStubsLinux.c
 *
 * This file contains implementations of common userspace functions in terms
 * that the Linux kernel can understand.
 */

/* Must come before any kernel header file */
#include "driver-config.h"
#include "kernelStubs.h"
#include "compat_kernel.h"
#include "compat_page.h"
#include "compat_sched.h"
#include <linux/slab.h>

#include "vm_assert.h"

/*
 *-----------------------------------------------------------------------------
 *
 * Panic --
 *
 *    Prints the debug message and stops the system.
 *
 * Results:
 *    None.
 *
 * Side effects:
 *    None
 *
 *-----------------------------------------------------------------------------
 */

void
Panic(const char *fmt, ...) // IN
{
   va_list args;
   char *result;

va_start(args, fmt);
   result = Str_Vasprintf(NULL, fmt, args);
   va_end(args);

if (result) {
      printk(KERN_EMERG "%s", result);
   }

BUG();

while (1); // Avoid compiler warning.
}

/*
 *----------------------------------------------------------------------
 *
 * Str_Strcpy--
 *
 *    Wrapper for strcpy that checks for buffer overruns.
 *
 * Results:
 *    Same as strcpy.
 *
 * Side effects:
 *    None.
 *
 *----------------------------------------------------------------------
 */

char *
Str_Strcpy(char *buf,       // OUT
           const char *src, // IN
           size_t maxSize)  // IN
{
   size_t len;

len = strlen(src);
   if (len >= maxSize) {
#ifdef GetReturnAddress
      Panic("%s:%d Buffer too small 0x%p\n", __FILE__, __LINE__, GetReturnAddress());
#else
      Panic("%s:%d Buffer too small\n", __FILE__, __LINE__);
#endif
   }
   return memcpy(buf, src, len + 1);
}

/*
 *----------------------------------------------------------------------
 *
 * Str_Vsnprintf --
 *
 *	Compatability wrapper b/w different libc versions
 *
 * Results:
 *	int - number of bytes written (not including NULL terminate character),
 *	      -1 on overflow (insufficient space for NULL terminate is considered
 *	      overflow)
 *
 *	NB: on overflow the buffer WILL be null terminated
 *
 * Side effects:
 *	None
 *
 *----------------------------------------------------------------------
 */

int
Str_Vsnprintf(char *str,          // OUT
              size_t size,        // IN
              const char *format, // IN
              va_list arguments)  // IN
{
   int retval;
   retval = vsnprintf(str, size, format, arguments);

/*
    * Linux glibc 2.0.x returns -1 and null terminates (which we shouldn't
    * be linking against), but glibc 2.1.x follows c99 and returns
    * characters that would have been written.
    */
   if (retval >= size) {
      return -1;
   }
   return retval;
}

/*
 *-----------------------------------------------------------------------------
 *
 * Str_Vasprintf --
 *
 *    Allocate and format a string, using the GNU libc way to specify the
 *    format (i.e. optionally allow the use of positional parameters)
 *
 * Results:
 *    The allocated string on success (if 'length' is not NULL, *length
 *       is set to the length of the allocated string)
 *    NULL on failure
 *
 * Side effects:
 *    None
 *
 *-----------------------------------------------------------------------------
 */

char *
Str_Vasprintf(size_t *length,       // OUT
              const char *format,   // IN
              va_list arguments)    // IN
{
   /*
    * Simple implementation of Str_Vasprintf when userlevel libraries are not
    * available (e.g. for use in drivers). We just fallback to vsnprintf,
    * doubling if we didn't have enough space.
    */
   unsigned int bufSize;
   char *buf;
   int retval;

bufSize = strlen(format);
   buf = NULL;

do {
      /*
       * Initial allocation of strlen(format) * 2. Should this be tunable?
       * XXX Yes, this could overflow and spin forever when you get near 2GB
       *     allocations. I don't care. --rrdharan
       */
      va_list args2;

bufSize *= 2;
      buf = realloc(buf, bufSize);

if (!buf) {
         return NULL;
      }

va_copy(args2, arguments);
      retval = Str_Vsnprintf(buf, bufSize, format, args2);
      va_end(args2);
   } while (retval == -1);

if (length) {
      *length = retval;
   }

/*
    * Try to trim the buffer here to save memory?
    */
   return buf;
}

/*
 *-----------------------------------------------------------------------------
 *
 * Str_Asprintf --
 *
 *    Same as Str_Vasprintf(), but parameters are passed inline --hpreg
 *
 * Results:
 *    Same as Str_Vasprintf()
 *
 * Side effects:
 *    Same as Str_Vasprintf()
 *
 *-----------------------------------------------------------------------------
 */

char *
Str_Asprintf(size_t *length,       // OUT
             const char *format,   // IN
             ...)                  // IN
{
   va_list arguments;
   char *result;

va_start(arguments, format);
   result = Str_Vasprintf(length, format, arguments);
   va_end(arguments);

return result;
}

/*
 *-----------------------------------------------------------------------------
 *
 * strdup --
 *
 *    Duplicates a string.
 *
 * Results:
 *    A pointer to memory containing the duplicated string or NULL if no
 *    memory was available.
 *
 * Side effects:
 *    None
 *
 *-----------------------------------------------------------------------------
 */

char *
strdup(const char *source) // IN
{
   char *target = NULL;
   if (source) {

/*
       * We call our special implementation of malloc() because the users of
       * strdup() will call free(), and that'll decrement the pointer before
       * freeing it. Thus, we need to make sure that the allocated block
       * also stores the block length before the block itself (see malloc()
       * below).
       */
      unsigned int len = strlen(source);
      target = malloc(len + 1);
      if (target) {
         memcpy(target, source, len + 1);
      }
   }

return target;
}

/*
 *----------------------------------------------------------------------------
 *
 * mallocReal --
 *
 *      Allocate memory using kmalloc. There is no realloc
 *      equivalent, so we roll our own by padding each allocation with
 *      4 (or 8 for 64 bit guests) extra bytes to store the block length.
 *
 * Results:
 *      Pointer to driver heap memory, offset by 4 (or 8)
 *      bytes from the real block pointer.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------------
 */

static void *
mallocReal(size_t size) // IN
{
   size_t *ptr;
   ptr = kmalloc(size + sizeof size, GFP_KERNEL);

if (ptr) {
      *ptr++ = size;
   }
   return ptr;
}

/*
 *----------------------------------------------------------------------------
 *
 * malloc --
 *
 *      Allocate memory using the common mallocReal.
 *
 * Note: This calls mallocReal and not malloc as the gcc 5.1.1 optimizer
 *       will replace the malloc and memset with a calloc call. This results
 *       in calloc calling itself and results in system crashes. See bug 1413226.
 *
 * Results:
 *      Pointer to driver heap memory, offset by 4 (or 8)
 *      bytes from the real block pointer.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------------
 */

void *
malloc(size_t size) // IN
{
   return mallocReal(size);
}

/*
 *---------------------------------------------------------------------------
 *
 * free --
 *
 *     Free memory allocated by a previous call to malloc, calloc or realloc.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     Calls kfree to free the real (base) pointer.
 *
 *---------------------------------------------------------------------------
 */

void
free(void *mem) // IN
{
   if (mem) {
      size_t *dataPtr = (size_t *)mem;
      kfree(--dataPtr);
   }
}

/*
 *----------------------------------------------------------------------------
 *
 * calloc --
 *
 *      Malloc and zero.
 *
 * Note: This calls mallocReal and not malloc as the gcc 5.1.1 optimizer
 *       will replace the malloc and memset with a calloc call. This results
 *       for system crashes when used by kernel components. See bug 1413226.
 *
 * Results:
 *      Pointer to driver heap memory (see malloc, above).
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------------
 */

void *
calloc(size_t num, // IN
       size_t len) // IN
{
   size_t size;
   void *ptr;

size = num * len;
   ptr = mallocReal(size);
   if (ptr) {
      memset(ptr, 0, size);
   }
   return ptr;
}

/*
 *----------------------------------------------------------------------------
 *
 * realloc --
 *
 *      Since the driver heap has no realloc equivalent, we have to roll our
 *      own. Fortunately, we can retrieve the block size of every block we
 *      hand out since we stashed it at allocation time (see malloc above).
 *
 * Results:
 *      Pointer to memory block valid for 'newSize' bytes, or NULL if
 *      allocation failed.
 *
 * Side effects:
 *      Could copy memory around.
 *
 *----------------------------------------------------------------------------
 */

void *
realloc(void* ptr,      // IN
        size_t newSize) // IN
{
   void *newPtr;
   size_t *dataPtr;
   size_t length, lenUsed;

dataPtr = (size_t *)ptr;
   length = ptr ? dataPtr[-1] : 0;
   if (newSize == 0) {
      if (ptr) {
         free(ptr);
         newPtr = NULL;
      } else {
         newPtr = malloc(newSize);
      }
   } else if (newSize == length) {
      newPtr = ptr;
   } else if ((newPtr = malloc(newSize))) {
      if (length < newSize) {
         lenUsed = length;
      } else {
         lenUsed = newSize;
      }
      memcpy(newPtr, ptr, lenUsed);
      free(ptr);
   }
   return newPtr;
}

�������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vmciKernelAPI1.h����������������������������������������������������������������0000444�0000000�0000000�00000020110�13432725350�016264� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2010,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmciKernelAPI1.h --
 *
 *    Kernel API (v1) exported from the VMCI host and guest drivers.
 */

#ifndef __VMCI_KERNELAPI_1_H__
#define __VMCI_KERNELAPI_1_H__

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#include "includeCheck.h"

#include "vmci_defs.h"
#include "vmci_call_defs.h"

#if defined __cplusplus
extern "C" {
#endif

/* VMCI module namespace on vmkernel. */

#define MOD_VMCI_NAMESPACE "com.vmware.vmci"

/* Define version 1. */

#undef  VMCI_KERNEL_API_VERSION
#define VMCI_KERNEL_API_VERSION_1 1
#define VMCI_KERNEL_API_VERSION   VMCI_KERNEL_API_VERSION_1

/* Macros to operate on the driver version number. */

#define VMCI_MAJOR_VERSION(v) (((v) >> 16) & 0xffff)
#define VMCI_MINOR_VERSION(v) ((v) & 0xffff)

#if defined(_WIN32)
/* Path to callback object in object manager, for Windows only. */
#define VMCI_CALLBACK_OBJECT_PATH L"\\Callback\\VMCIDetachCB"
#endif // _WIN32

/* VMCI Device Usage API. */

#if defined(__linux__) && !defined(VMKERNEL)
#define vmci_device_get(_a, _b, _c, _d) 1
#define vmci_device_release(_x)
#else // !linux
typedef void (VMCI_DeviceShutdownFn)(void *deviceRegistration,
                                     void *userData);
Bool vmci_device_get(uint32 *apiVersion,
                     VMCI_DeviceShutdownFn *deviceShutdownCB,
                     void *userData, void **deviceRegistration);
void vmci_device_release(void *deviceRegistration);
#endif // !linux

#if defined(_WIN32)
/* Called when the client is unloading, for Windows only. */
void vmci_exit(void);
#endif // _WIN32

/* VMCI Datagram API. */

int vmci_datagram_create_handle(uint32 resourceId, uint32 flags,
                                VMCIDatagramRecvCB recvCB, void *clientData,
                                VMCIHandle *outHandle);
int vmci_datagram_create_handle_priv(uint32 resourceID, uint32 flags,
                                     VMCIPrivilegeFlags privFlags,
                                     VMCIDatagramRecvCB recvCB,
                                     void *clientData, VMCIHandle *outHandle);
int vmci_datagram_destroy_handle(VMCIHandle handle);
int vmci_datagram_send(VMCIDatagram *msg);

/* VMCI Utility API. */

VMCIId vmci_get_context_id(void);

#if defined(__linux__) && !defined(VMKERNEL)
/* Returned value is a bool, 0 for false, 1 for true. */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
int vmci_is_context_owner(VMCIId contextID, kuid_t uid);
#else
int vmci_is_context_owner(VMCIId contextID, uid_t uid);
#endif
#else // !linux || VMKERNEL
/* Returned value is a VMCI error code. */
int vmci_is_context_owner(VMCIId contextID, void *hostUser);
#endif // !linux || VMKERNEL

uint32 vmci_version(void);
int vmci_cid_2_host_vm_id(VMCIId contextID, void *hostVmID,
                          size_t hostVmIDLen);

/* VMCI Event API. */

typedef void (*VMCI_EventCB)(VMCIId subID, VMCI_EventData *ed,
                             void *clientData);

int vmci_event_subscribe(VMCI_Event event,
#if !defined(__linux__) && !defined(__FreeBSD__) || defined(VMKERNEL)
                         uint32 flags,
#endif // !linux && !FreeBSD || VMKERNEL
                         VMCI_EventCB callback,
                         void *callbackData, VMCIId *subID);
int vmci_event_unsubscribe(VMCIId subID);

/* VMCI Context API */

VMCIPrivilegeFlags vmci_context_get_priv_flags(VMCIId contextID);

/* VMCI Queue Pair API. */

typedef struct VMCIQPair VMCIQPair;

int vmci_qpair_alloc(VMCIQPair **qpair, VMCIHandle *handle,
                     uint64 produceQSize, uint64 consumeQSize, VMCIId peer,
                     uint32 flags, VMCIPrivilegeFlags privFlags);
int vmci_qpair_detach(VMCIQPair **qpair);
int vmci_qpair_get_produce_indexes(const VMCIQPair *qpair,
                                   uint64 *producerTail, uint64 *consumerHead);
int vmci_qpair_get_consume_indexes(const VMCIQPair *qpair,
                                   uint64 *consumerTail, uint64 *producerHead);
int64 vmci_qpair_produce_free_space(const VMCIQPair *qpair);
int64 vmci_qpair_produce_buf_ready(const VMCIQPair *qpair);
int64 vmci_qpair_consume_free_space(const VMCIQPair *qpair);
int64 vmci_qpair_consume_buf_ready(const VMCIQPair *qpair);
ssize_t vmci_qpair_enqueue(VMCIQPair *qpair, const void *buf, size_t bufSize,
                           int mode);
ssize_t vmci_qpair_dequeue(VMCIQPair *qpair, void *buf, size_t bufSize,
                           int mode);
ssize_t vmci_qpair_peek(VMCIQPair *qpair, void *buf, size_t bufSize, int mode);

#if (defined(__APPLE__) && !defined (VMX86_TOOLS)) || \
    (defined(__linux__) && defined(__KERNEL__))    || \
    (defined(_WIN32)    && defined(WINNT_DDK))
/*
 * Environments that support struct iovec
 */

ssize_t vmci_qpair_enquev(VMCIQPair *qpair, void *iov, size_t iovSize,
                          int mode);
ssize_t vmci_qpair_dequev(VMCIQPair *qpair, void *iov, size_t iovSize,
                          int mode);
ssize_t vmci_qpair_peekv(VMCIQPair *qpair, void *iov, size_t iovSize,
                         int mode);
#endif /* Systems that support struct iovec */

/* Typedefs for all of the above, used by the IOCTLs and the kernel library. */

typedef void (VMCI_DeviceReleaseFct)(void *);
typedef int (VMCIDatagram_CreateHndFct)(VMCIId, uint32, VMCIDatagramRecvCB,
                                        void *, VMCIHandle *);
typedef int (VMCIDatagram_CreateHndPrivFct)(VMCIId, uint32, VMCIPrivilegeFlags,
                                            VMCIDatagramRecvCB, void *,
                                            VMCIHandle *);
typedef int (VMCIDatagram_DestroyHndFct)(VMCIHandle);
typedef int (VMCIDatagram_SendFct)(VMCIDatagram *);
typedef VMCIId (VMCI_GetContextIDFct)(void);
typedef uint32 (VMCI_VersionFct)(void);
typedef int (VMCI_ContextID2HostVmIDFct)(VMCIId, void *, size_t);
typedef int (VMCI_IsContextOwnerFct)(VMCIId, void *);
typedef int (VMCIEvent_SubscribeFct)(VMCI_Event, uint32, VMCI_EventCB, void *,
                                     VMCIId *);
typedef int (VMCIEvent_UnsubscribeFct)(VMCIId);
typedef VMCIPrivilegeFlags (VMCIContext_GetPrivFlagsFct)(VMCIId);
typedef int (VMCIQPair_AllocFct)(VMCIQPair **, VMCIHandle *, uint64, uint64,
                                 VMCIId, uint32, VMCIPrivilegeFlags);
typedef int (VMCIQPair_DetachFct)(VMCIQPair **);
typedef int (VMCIQPair_GetProduceIndexesFct)(const VMCIQPair *, uint64 *,
                                             uint64 *);
typedef int (VMCIQPair_GetConsumeIndexesFct)(const VMCIQPair *, uint64 *,
                                             uint64 *);
typedef int64 (VMCIQPair_ProduceFreeSpaceFct)(const VMCIQPair *);
typedef int64 (VMCIQPair_ProduceBufReadyFct)(const VMCIQPair *);
typedef int64 (VMCIQPair_ConsumeFreeSpaceFct)(const VMCIQPair *);
typedef int64 (VMCIQPair_ConsumeBufReadyFct)(const VMCIQPair *);
typedef ssize_t (VMCIQPair_EnqueueFct)(VMCIQPair *, const void *, size_t, int);
typedef ssize_t (VMCIQPair_DequeueFct)(VMCIQPair *, void *, size_t, int);
typedef ssize_t (VMCIQPair_PeekFct)(VMCIQPair *, void *, size_t, int);
typedef ssize_t (VMCIQPair_EnqueueVFct)(VMCIQPair *qpair, void *, size_t, int);
typedef ssize_t (VMCIQPair_DequeueVFct)(VMCIQPair *qpair, void *, size_t, int);
typedef ssize_t (VMCIQPair_PeekVFct)(VMCIQPair *qpair, void *, size_t, int);

#if defined __cplusplus
} // extern "C"
#endif

#endif /* !__VMCI_KERNELAPI_1_H__ */

��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vmci_infrastructure.h�����������������������������������������������������������0000444�0000000�0000000�00000010362�13432725350�017660� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006,2014 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmci_infrastructure.h --
 *
 *      This file implements the VMCI infrastructure.
 */

#ifndef _VMCI_INFRASTRUCTURE_H_
#define _VMCI_INFRASTRUCTURE_H_

#define INCLUDE_ALLOW_USERLEVEL

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#include "includeCheck.h"

#include "vmware.h"
#include "vmci_defs.h"

#if defined __cplusplus
extern "C" {
#endif

typedef enum {
   VMCIOBJ_VMX_VM = 10,
   VMCIOBJ_CONTEXT,
   VMCIOBJ_SOCKET,
   VMCIOBJ_NOT_SET,
} VMCIObjType;

/* For storing VMCI structures in file handles. */
typedef struct VMCIObj {
   void *ptr;
   VMCIObjType type;
} VMCIObj;

/* Guestcalls currently support a maximum of 8 uint64 arguments. */
#define VMCI_GUESTCALL_MAX_ARGS_SIZE 64

/*
 * Structure used for checkpointing the doorbell mappings. It is
 * written to the checkpoint as is, so changing this structure will
 * break checkpoint compatibility.
 */

typedef struct VMCIDoorbellCptState {
   VMCIHandle handle;
   uint64     bitmapIdx;
} VMCIDoorbellCptState;

/* Used to determine what checkpoint state to get and set. */
#define VMCI_NOTIFICATION_CPT_STATE          0x1
#define VMCI_WELLKNOWN_CPT_STATE             0x2
#define VMCI_DG_OUT_STATE                    0x3
#define VMCI_DG_IN_STATE                     0x4
#define VMCI_DG_IN_SIZE_STATE                0x5
#define VMCI_DOORBELL_CPT_STATE              0x6
#define VMCI_DG_HYPERVISOR_SAVE_STATE_SIZE   0x7
#define VMCI_DG_HYPERVISOR_SAVE_STATE        0x8

/* Used to control the VMCI device in the vmkernel */
#define VMCI_DEV_RESET            0x01
#define VMCI_DEV_QP_RESET         0x02  // DEPRECATED
#define VMCI_DEV_QUIESCE          0x03
#define VMCI_DEV_UNQUIESCE        0x04
#define VMCI_DEV_QP_BREAK_SHARING 0x05  // DEPRECATED
#define VMCI_DEV_RESTORE_SYNC     0x06
#define VMCI_DEV_BMASTER_OFF      0x07
#define VMCI_DEV_BMASTER_ON       0x08

/*
 *-------------------------------------------------------------------------
 *
 *  VMCI_Hash --
 *
 *     Hash function used by the Simple Datagram API. Based on the djb2
 *     hash function by Dan Bernstein.
 *
 *  Result:
 *     Returns guest call size.
 *
 *  Side effects:
 *     None.
 *
 *-------------------------------------------------------------------------
 */

static INLINE int
VMCI_Hash(VMCIHandle handle, // IN
          unsigned size)     // IN
{
   unsigned     i;
   int          hash        = 5381;
   const uint64 handleValue = QWORD(handle.resource, handle.context);

for (i = 0; i < sizeof handle; i++) {
      hash = ((hash << 5) + hash) + (uint8)(handleValue >> (i * 8));
   }
   return hash & (size - 1);
}

/*
 *-------------------------------------------------------------------------
 *
 *  VMCI_HashId --
 *
 *     Hash function used by the Simple Datagram API. Hashes only a VMCI id
 *     (not the full VMCI handle) Based on the djb2
 *     hash function by Dan Bernstein.
 *
 *  Result:
 *     Returns guest call size.
 *
 *  Side effects:
 *     None.
 *
 *-------------------------------------------------------------------------
 */

static INLINE int
VMCI_HashId(VMCIId id,      // IN
            unsigned size)  // IN
{
   unsigned     i;
   int          hash        = 5381;

for (i = 0; i < sizeof id; i++) {
      hash = ((hash << 5) + hash) + (uint8)(id >> (i * 8));
   }
   return hash & (size - 1);
}

#if defined __cplusplus
} // extern "C"
#endif

#endif // _VMCI_INFRASTRUCTURE_H_
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/�����������������������������������������������������������������������0000755�0000000�0000000�00000000000�13432726376�015240� 5����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/file_operations_fsync.c������������������������������������������������0000444�0000000�0000000�00000002746�13432725347�021774� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2011 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * Linux v3.1 added 2 params to fsync for fine-grained locking control.
 * But SLES11 SP2 has backported the change to its 3.0 kernel,
 * so we can't rely solely on kernel version to determine number of
 * arguments.
 */

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0)
#   error This compile test intentionally fails.
#else

#include <linux/fs.h>
#include <linux/types.h>  /* loff_t */

static int TestFsync(struct file *file,
                     loff_t start, loff_t end,
                      int datasync)
{
   return 0;
}

struct file_operations testFO = {
   .fsync = TestFsync,
};

#endif
��������������������������vmxnet3-only/shared/autoconf/getsb1.c���������������������������������������������������������������0000444�0000000�0000000�00000003076�13432725347�016572� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)

#include <linux/fs.h>

/*
 * Around 2.6.18, a pointer to a vfsmount was added to get_sb. Red Hat 
 * backported this behavior into a 2.6.17 kernel.
 *
 * This test will fail on a kernel with such a patch.
 */
static struct super_block * LinuxDriverGetSb(struct file_system_type *fs_type,
                                             int flags,
                                             const char *dev_name,
                                             void *rawData)
{
   return 0;
}

struct file_system_type fs_type = {
   .get_sb = LinuxDriverGetSb
};
#else
#error "This test intentionally fails on 2.6.19 or newer kernels."
#endif
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/file_operations_flush.c������������������������������������������������0000444�0000000�0000000�00000002664�13432725347�021772� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2013-2014 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * Linux v2.6.18 added an owner parameter to flush.
 * But SLES10 has backported the change to its 2.6.16.60 kernel,
 * so we can't rely solely on kernel version to determine number of
 * arguments.
 *
 * This test will fail on a kernel with such a patch.
 */

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
#error This compile test intentionally fails on 2.6.18 and newer kernels.
#else

#include <linux/fs.h>

static int TestFlush(struct file *file);
{
   return 0;
}

struct file_operations testFO = {
   .flush = TestFlush,
};

#endif
����������������������������������������������������������������������������vmxnet3-only/shared/autoconf/netif_num_params.c�����������������������������������������������������0000444�0000000�0000000�00000003352�13432725347�020731� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2009 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * Detect whether netif_rx_complete (and netif_rx_schedule) take a single
 * napi_struct argument. The foundation was laid whith introducing Generic
 * Receive Offload infrastructure but dropping unneeded net_device argument
 * did not happen till few commits later so we can't simply test for presence
 * of NETIF_F_GRO.
 *
 * Test succeeds if netif_rx_complete takes dev & napi arguments, or if it
 * takes dev argument only (kernels before 2.6.24).  Test fails if netif_rx_complete
 * takes only single napi argument.
 */

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 30)
#   error This compile test intentionally fails.
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
#include <linux/netdevice.h>

#ifdef NETIF_F_GRO
void test_netif_rx_complete(struct net_device *dev, struct napi_struct *napi)
{
   netif_rx_complete(dev, napi);
}
#endif

#include "compat_version.h"
#include "compat_autoconf.h"

/*
 * Between 2.6.23 and 2.6.24-rc1 ctor prototype was changed from
 * ctor(ptr, cache, flags) to ctor(cache, ptr).  Unfortunately there
 * is no typedef for ctor, so we have to redefine kmem_cache_create
 * to find out ctor prototype.  This assumes that kmem_cache_create
 * takes 5 arguments and not 6 - that change occured between
 * 2.6.22 and 2.6.23-rc1.  If prototype matches, then this is old
 * kernel.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
#error "This test intentionally fails on 2.6.24 and newer kernels."
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23)
#include <linux/slab.h>

struct kmem_cache *kmem_cache_create(const char *, size_t, size_t,
                        unsigned long,
                        void (*)(void *, struct kmem_cache *, unsigned long));
						
#endif
����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/geninclude.c�����������������������������������������������������������0000444�0000000�0000000�00000002321�13432725347�017512� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2003 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "compat_version.h"
#include "compat_autoconf.h"

#ifdef CONFIG_X86_VOYAGER
APATH/mach-voyager
#endif
#ifdef CONFIG_X86_VISWS
APATH/mach-visws
#endif
#ifdef CONFIG_X86_NUMAQ
APATH/mach-numaq
#endif
#ifdef CONFIG_X86_BIGSMP
APATH/mach-bigsmp
#endif
#ifdef CONFIG_X86_SUMMIT
APATH/mach-summit
#endif
#ifdef CONFIG_X86_GENERICARCH
APATH/mach-generic
#endif
APATH/mach-default

���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/cachector1.c�����������������������������������������������������������0000444�0000000�0000000�00000003075�13432725347�017420� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2008 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "compat_version.h"
#include "compat_autoconf.h"

/*
 * Between 2.6.27-rc1 and 2.6.27-rc2 ctor prototype was changed from
 * ctor(cache, ptr) to ctor(ptr).  Unfortunately there
 * is no typedef for ctor, so we have to redefine kmem_cache_create
 * to find out ctor prototype.  If prototype matches, then this is old
 * kernel.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
#error "This test intentionally fails on 2.6.28 and newer kernels."
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
#include <linux/slab.h>

struct kmem_cache *kmem_cache_create(const char *, size_t, size_t,
                                     unsigned long,
                                     void (*)(struct kmem_cache *, void *));
						
#endif
�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/wait_on_bit.c����������������������������������������������������������0000444�0000000�0000000�00000003306�13432725347�017677� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) && \
    LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 13)

#include <linux/fs.h>
#include <linux/wait.h>
#include <linux/sched.h>

unsigned long test_bits;

/*
 * After 3.17.0, wait_on_bit changed its interface to remove the action
 * callback argument and this was backported to some Linux kernel versions
 * such as 3.10 for the RHEL 7.3 version.
 *
 * This test will fail on a kernel with such a patch.
 */

int test(void)
{

return wait_on_bit(&test_bits,
                      0,
                      NULL,
                      TASK_UNINTERRUPTIBLE);
}
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)
#error "This test intentionally fails on 3.17.0 and newer kernels."
#else
/*
 * It must be older than 2.6.13 in which case we don't use the function.
 */
#endif
��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/cachecreate.c����������������������������������������������������������0000444�0000000�0000000�00000003210�13432725347�017622� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "compat_version.h"
#include "compat_autoconf.h"

/*
 * All kernels before 2.6.22 take 6 arguments.  All kernels since
 * 2.6.23-rc1 take 5 arguments.  Only kernels between 2.6.22 and
 * 2.6.23-rc1 are questionable - we could ignore them if we wanted,
 * nobody cares about them even now.  But unfortunately RedHat is
 * re-releasing 2.6.X-rc kernels under 2.6.(X-1) name, so they
 * are releasing 2.6.23-rc1 as 2.6.22-5055-something, so we have
 * to do autodetection for them.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 22)
/* Success... */
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23)
#error "This test intentionally fails on 2.6.23 and newer kernels."
#else
#include <linux/slab.h>

struct kmem_cache *kmemtest(void) {
   return kmem_cache_create("test", 12, 0, 0, NULL, NULL);
}
						
#endif
����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/skblin.c���������������������������������������������������������������0000444�0000000�0000000�00000002572�13432725347�016667� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * Detect whether skb_linearize takes one or two arguments.
 */

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE <= KERNEL_VERSION(2, 6, 17)
/*
 * Since 2.6.18 all kernels have single-argument skb_linearize.  For
 * older kernels use autodetection.  Not using autodetection on newer
 * kernels saves us from compile failure on some post 2.6.18 kernels
 * which do not have selfcontained skbuff.h.
 */

#include <linux/skbuff.h>

int test_skb_linearize(struct sk_buff *skb)
{
   return skb_linearize(skb);
}

#endif
��������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/dalias.c���������������������������������������������������������������0000444�0000000�0000000�00000003217�13432725347�016637� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2015-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0)
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0)
#include <linux/dcache.h>
#include <linux/list.h>

/*
 * After 3.19.0, the dentry d_alias field was moved. Fedora
 * backported this behavior into earlier kernel versions.
 * The type of the d_alias field changed from 3.6 onwards
 * which was a list head to being a list node. The check
 * for earlier than 3.6 is done separately.
 *
 * This test will fail on a kernel with such a patch.
 */
void test(void)
{
   struct dentry aliasDentry;

INIT_HLIST_NODE(&aliasDentry.d_alias);
}

#else
/* Intentionally passes for earlier than 3.6.0 kernels as a separate test is done. */
#endif
#else
#error "This test intentionally fails on 3.19.0 or newer kernels."
#endif
���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/truncate_pagecache.c���������������������������������������������������0000444�0000000�0000000�00000003315�13432725347�021206� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2015-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && \
    LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32)

#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/types.h>  /* loff_t */

/*
 * After 3.12.0, truncate_pagecache changed its interface to just use
 * the new file size only. Red Hat backported this behavior into a 3.10.0
 * kernel.
 *
 * This test will fail on a kernel with such a patch.
 */

void test(void)
{
   struct inode inode;
   loff_t oldSize = 0;
   loff_t newSize = 4096;

truncate_pagecache(&inode, oldSize, newSize);
}
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)
#error "This test intentionally fails on 3.12.0 and newer kernels."
#else
/*
 * It must be older than 2.6.32 in which case we assume success.
 * So not 3.12 compatible. There is no function for these versions.
 */
#endif
�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/netcreate_num_params.c�������������������������������������������������0000444�0000000�0000000�00000003160�13432725347�021573� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2010 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * During 2.6.33 merge window net_proto_ops->create() method was changed -
 * a new 'kern' field, signalling whether socket is being created by kernel
 * or userspace application, was added to it. Unfortunately, some
 * distributions, such as RHEL 6, have backported the change to earlier
 * kernels, so we can't rely solely on kernel version to determine number of
 * arguments.
 */

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
#   error This compile test intentionally fails.
#else

#include <linux/net.h>

static int TestCreate(struct net *net,
                      struct socket *sock, int protocol,
                      int kern)
{
   return 0;
}

struct net_proto_family testFamily = {
   .create = TestCreate,
};

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20)
#include <linux/fs.h>
#include <linux/types.h>  /* loff_t */
#include <linux/stddef.h> /* NULL */

/*
 * After 2.6.18, filldir and statfs were changed to send 64-bit inode
 * numbers to user space. Red Hat backported this behavior into a 2.6.17
 * kernel.
 *
 * This test will fail on a kernel with such a patch.
 */
static int LinuxDriverFilldir(void *buf,
                              const char *name,
                              int namelen,
                              loff_t offset,
                              ino_t ino,
                              unsigned int d_type)
{
   return 0;
}

void test(void)
{
   vfs_readdir(NULL, LinuxDriverFilldir, NULL);
}
#else
#error "This test intentionally fails on 2.6.20 and newer kernels."
#endif
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/statfs1.c��������������������������������������������������������������0000444�0000000�0000000�00000002671�13432725347�016772� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
#include <linux/fs.h>

/*
 * Around 2.6.18, the super_block pointer in statfs was changed to a dentry
 * pointer. Red Hat backported this behavior into a 2.6.17 kernel.
 *
 * This test will fail on a kernel with such a patch.
 */
static int LinuxDriverStatFs(struct super_block *sb,
                             struct kstatfs *stat) 
{
   return 0;
}

struct super_operations super_ops = {
   .statfs = LinuxDriverStatFs
};
#else
#error "This test intentionally fails on 2.6.19 and newer kernels."
#endif
�����������������������������������������������������������������������vmxnet3-only/shared/autoconf/dcount.c���������������������������������������������������������������0000444�0000000�0000000�00000002603�13432725347�016674� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2014 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0)
#include <linux/dcache.h>

/*
 * After 3.11.0, the dentry d_count field was removed. Red Hat
 * backported this behavior into a 3.10.0 kernel.
 *
 * This test will fail on a kernel with such a patch.
 */
void test(void)
{
   struct dentry dentry;

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 38)
   dentry.d_count = 1;
#else
   atomic_set(&dentry.d_count, 1);
#endif
}
#else
#error "This test intentionally fails on 3.11.0 or newer kernels."
#endif
�����������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/dalias1.c��������������������������������������������������������������0000444�0000000�0000000�00000003325�13432725347�016720� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2015-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 6, 0) && \
    LINUX_VERSION_CODE >= KERNEL_VERSION(3, 2, 0)
#include <linux/dcache.h>
#include <linux/list.h>

/*
 * After 3.19.0, the dentry d_alias field was moved. Fedora
 * backported this behavior into earlier kernels.
 * The type of the d_alias field changed from 3.6 onwards
 * which was a list head to being a list node. The check
 * for 3.6 onwards is done separately.
 *
 * This test will fail on a kernel with such a patch.
 */
void test(void)
{
   struct dentry aliasDentry;

INIT_LIST_HEAD(&aliasDentry.d_alias);
}

#else
/*
 * Intentionally passes for earlier than 3.2.0 kernels as d_alias is valid.
 *
 * Intentionally passes for 3.6.0 or later kernels as d_alias is a different type.
 * A separate test with the different type is run for those kernel versions.
 */
#endif
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/autoconf/inode1.c���������������������������������������������������������������0000444�0000000�0000000�00000002703�13432725347�016560� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#include "compat_version.h"
#include "compat_autoconf.h"

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20)
#include <linux/fs.h>
#include <linux/stddef.h> /* NULL */

/*
 * After 2.6.18, inodes were "slimmed". This involved removing the union
 * that encapsulates inode private data (and using i_private instead), as well
 * as removing i_blksize. Red Hat backported this behavior into a 2.6.17
 * kernel.
 *
 * This test will fail on a kernel with such a patch.
 */
void test(void)
{
   struct inode inode;

inode.u.generic_ip = NULL;
}
#else
#error "This test intentionally fails on 2.6.20 and newer kernels."
#endif
�������������������������������������������������������������vmxnet3-only/shared/compat_pagemap.h����������������������������������������������������������������0000444�0000000�0000000�00000002535�13432725347�016550� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2009 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_PAGEMAP_H__
#   define __COMPAT_PAGEMAP_H__

#include <linux/pagemap.h>

/*
 * AOP_FLAG_NOFS was defined in the same changeset that
 * grab_cache_page_write_begin() was introduced.
 */
#ifdef AOP_FLAG_NOFS
#define compat_grab_cache_page_write_begin(mapping, index, flags) \
         grab_cache_page_write_begin((mapping), (index), (flags))
#else
#define compat_grab_cache_page_write_begin(mapping, index, flags) \
         __grab_cache_page((mapping), (index));
#endif

#endif /* __COMPAT_PAGEMAP_H__ */
�������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_mutex.h������������������������������������������������������������������0000444�0000000�0000000�00000003475�13432725347�016304� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2009 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_MUTEX_H__
#   define __COMPAT_MUTEX_H__

/* Blocking mutexes were introduced in 2.6.16. */

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 16)

#include "compat_semaphore.h"

typedef struct semaphore compat_mutex_t;

# define compat_define_mutex(_mx)               DECLARE_MUTEX(_mx)
# define compat_mutex_init(_mx)                 init_MUTEX(_mx)
# define compat_mutex_lock(_mx)                 down(_mx)
# define compat_mutex_lock_interruptible(_mx)   down_interruptible(_mx)
# define compat_mutex_unlock(_mx)               up(_mx)

#else

#include <linux/mutex.h>

typedef struct mutex compat_mutex_t;

# define compat_define_mutex(_mx)               DEFINE_MUTEX(_mx)
# define compat_mutex_init(_mx)                 mutex_init(_mx)
# define compat_mutex_lock(_mx)                 mutex_lock(_mx)
# define compat_mutex_lock_interruptible(_mx)   mutex_lock_interruptible(_mx)
# define compat_mutex_unlock(_mx)               mutex_unlock(_mx)

#endif

#endif /* __COMPAT_MUTEX_H__ */
���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_string.h�����������������������������������������������������������������0000444�0000000�0000000�00000003563�13432725347�016446� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_STRING_H__
#   define __COMPAT_STRING_H__

#include <linux/string.h>

/*
 * kstrdup was born in 2.6.13. This implementation is almost identical to the
 * one found there.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 13)
#define compat_kstrdup(s, gfp) kstrdup(s, gfp)
#else
#define compat_kstrdup(s, gfp)                                                \
({                                                                            \
   size_t len;                                                                \
   char *buf;                                                                 \
   len = strlen(s) + 1;                                                       \
   buf = kmalloc(len, gfp);                                                   \
   memcpy(buf, s, len);                                                       \
   buf;                                                                       \
})                                                                            
#endif

#endif /* __COMPAT_STRING_H__ */
���������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vmciKernelAPI2.h����������������������������������������������������������������0000444�0000000�0000000�00000004240�13432725350�016273� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2010 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmciKernelAPI2.h --
 *
 *    Kernel API (v2) exported from the VMCI host and guest drivers.
 */

#ifndef __VMCI_KERNELAPI_2_H__
#define __VMCI_KERNELAPI_2_H__

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#include "includeCheck.h"

#include "vmciKernelAPI1.h"

#if defined __cplusplus
extern "C" {
#endif

/* Define version 2. */

#undef  VMCI_KERNEL_API_VERSION
#define VMCI_KERNEL_API_VERSION_2 2
#define VMCI_KERNEL_API_VERSION   VMCI_KERNEL_API_VERSION_2

/* VMCI Doorbell API. */

#define VMCI_FLAG_DELAYED_CB 0x01

typedef void (*VMCICallback)(void *clientData);

int vmci_doorbell_create(VMCIHandle *handle, uint32 flags,
                         VMCIPrivilegeFlags privFlags, VMCICallback notifyCB,
                         void *clientData);
int vmci_doorbell_destroy(VMCIHandle handle);
int vmci_doorbell_notify(VMCIHandle handle, VMCIPrivilegeFlags privFlags);

/* Typedefs for all of the above, used by the IOCTLs and the kernel library. */

typedef int (VMCIDoorbell_CreateFct)(VMCIHandle *, uint32, VMCIPrivilegeFlags,
                                     VMCICallback, void *);
typedef int (VMCIDoorbell_DestroyFct)(VMCIHandle);
typedef int (VMCIDoorbell_NotifyFct)(VMCIHandle, VMCIPrivilegeFlags);

#if defined __cplusplus
} // extern "C"
#endif

#endif /* !__VMCI_KERNELAPI_2_H__ */
����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vmci_kernel_if.h����������������������������������������������������������������0000444�0000000�0000000�00000042051�13432725350�016536� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006-2016,2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmci_kernel_if.h --
 *
 *      This file defines helper functions for VMCI host _and_ guest
 *      kernel code. It must work for Windows, Mac OS, vmkernel, Linux and
 *      Solaris kernels, i.e. using defines where necessary.
 */

#ifndef _VMCI_KERNEL_IF_H_
#define _VMCI_KERNEL_IF_H_

#if !defined(__linux__) && !defined(_WIN32) && !defined(__APPLE__) && \
    !defined(VMKERNEL)
#  error "Platform not supported."
#endif

#if defined(_WIN32)
#  include <ntddk.h>
#else
#define UNREFERENCED_PARAMETER(P)
#endif

#if defined(__linux__) && !defined(VMKERNEL)
#  include "driver-config.h"
#  include "compat_cred.h"
#  include "compat_module.h"
#  include "compat_semaphore.h"
#  include "compat_spinlock.h"
#  include "compat_version.h"
#  include <linux/wait.h>
#endif // linux

#ifdef __APPLE__
#  include <IOKit/IOLib.h>
#  include <mach/task.h>
#  include <mach/semaphore.h>
#  include <sys/kauth.h>
#endif

#ifdef VMKERNEL
#  include "splock.h"
#  include "splock_customRanks.h"
#  include "semaphore_ext.h"
#  include "vmkapi.h"
#  include "world_dist.h"
#endif

#include "vm_basic_types.h"
#include "vmci_defs.h"

#if defined(VMKERNEL)
#  include "list.h"
#else
#  include "dbllnklst.h"
#endif

#if defined __cplusplus
extern "C" {
#endif

/* Flags for specifying memory type. */
#define VMCI_MEMORY_NORMAL   0x0
#define VMCI_MEMORY_ATOMIC   0x1
#define VMCI_MEMORY_NONPAGED 0x2

/* Platform specific type definitions. */

#if defined(VMKERNEL)
#  define VMCI_EXPORT_SYMBOL(_SYMBOL)  VMK_MODULE_EXPORT_SYMBOL(_SYMBOL);
#elif defined(__linux__)
#  define VMCI_EXPORT_SYMBOL(_symbol)  EXPORT_SYMBOL(_symbol);
#elif defined(__APPLE__)
#  define VMCI_EXPORT_SYMBOL(_symbol)  __attribute__((visibility("default")))
#else
#  define VMCI_EXPORT_SYMBOL(_symbol)
#endif

#if defined(VMKERNEL)
  typedef MCSLock VMCILock;
  typedef SP_IRQL VMCILockFlags;
  typedef Semaphore VMCIEvent;
  typedef Semaphore VMCIMutex;
  typedef World_ID VMCIHostVmID;
  typedef uint32 VMCIHostUser;
  typedef PPN64 *VMCIQPGuestMem;
#elif defined(__linux__)
  typedef spinlock_t VMCILock;
  typedef unsigned long VMCILockFlags;
  typedef wait_queue_head_t VMCIEvent;
  typedef struct semaphore VMCIMutex;
  typedef PPN *VMCIPpnList; /* List of PPNs in produce/consume queue. */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)
  typedef kuid_t VMCIHostUser;
#else
  typedef uid_t VMCIHostUser;
#endif
  typedef VA64 VMCIQPGuestMem;
#elif defined(__APPLE__)
  typedef IOLock *VMCILock;
  typedef unsigned long VMCILockFlags;
  typedef struct {
     IOLock *lock;
     DblLnkLst_Links waiters;
     int buffered;
  } VMCIEvent;
  typedef IOLock *VMCIMutex;
  typedef void *VMCIPpnList; /* Actually a pointer to the C++ Object IOMemoryDescriptor */
  typedef uid_t VMCIHostUser;
  typedef VA64 *VMCIQPGuestMem;
#elif defined(_WIN32)
  typedef KSPIN_LOCK VMCILock;
  typedef KIRQL VMCILockFlags;
  typedef KEVENT VMCIEvent;
  typedef FAST_MUTEX VMCIMutex;
  typedef PMDL VMCIPpnList; /* MDL to map the produce/consume queue. */
  typedef PSID VMCIHostUser;
  typedef VA64 *VMCIQPGuestMem;
#endif // VMKERNEL

/* Callback needed for correctly waiting on events. */
typedef int (*VMCIEventReleaseCB)(void *clientData);

/*
 * Internal locking dependencies within VMCI:
 * * CONTEXTFIRE < CONTEXT, CONTEXTLIST, EVENT, HASHTABLE
 * * DOORBELL < HASHTABLE
 * * QPHIBERNATE < EVENT
 */

#ifdef VMKERNEL
  typedef Lock_Rank VMCILockRank;
  typedef SemaRank VMCISemaRank;

#define VMCI_SEMA_RANK_QPHEADER       (SEMA_RANK_FS - 1)

#define VMCI_LOCK_RANK_MAX_NONBLOCK   (MIN(SP_RANK_WAIT, \
                                             SP_RANK_HEAPLOCK_DYNAMIC) - 1)
  #define VMCI_LOCK_RANK_MAX            (SP_RANK_BLOCKABLE_HIGHEST_MAJOR - 2)

/*
   * Determines whether VMCI locks will be blockable or not. If blockable,
   * all locks will be at or below VMCI_LOCK_RANK_MAX. If not, locks will
   * instead use VMCI_LOCK_RANK_MAX_NONBLOCK as the maximum. The other
   * VMCI_LOCK_RANK_XXX values will be rebased to be non-blocking as well
   * in that case.
   */
  extern Bool vmciBlockableLock;
#else
  typedef unsigned long VMCILockRank;
  typedef unsigned long VMCISemaRank;

#define VMCI_LOCK_RANK_MAX            0x0fff

#define VMCI_SEMA_RANK_QPHEADER       0x0fff
#endif // VMKERNEL
#define VMCI_LOCK_RANK_CONTEXT          VMCI_LOCK_RANK_MAX
#define VMCI_LOCK_RANK_CONTEXTLIST      VMCI_LOCK_RANK_MAX
#define VMCI_LOCK_RANK_DATAGRAMVMK      VMCI_LOCK_RANK_MAX
#define VMCI_LOCK_RANK_EVENT            VMCI_LOCK_RANK_MAX
#define VMCI_LOCK_RANK_HASHTABLE        VMCI_LOCK_RANK_MAX
#define VMCI_LOCK_RANK_RESOURCE         VMCI_LOCK_RANK_MAX
#define VMCI_LOCK_RANK_QPHEADER         VMCI_LOCK_RANK_MAX
#define VMCI_LOCK_RANK_DOORBELL         (VMCI_LOCK_RANK_HASHTABLE - 1)
#define VMCI_LOCK_RANK_CONTEXTFIRE      (MIN(VMCI_LOCK_RANK_CONTEXT, \
                                         MIN(VMCI_LOCK_RANK_CONTEXTLIST, \
                                         MIN(VMCI_LOCK_RANK_EVENT, \
                                             VMCI_LOCK_RANK_HASHTABLE))) - 1)
#define VMCI_LOCK_RANK_QPHIBERNATE      (VMCI_LOCK_RANK_EVENT - 1)
#define VMCI_LOCK_RANK_PACKET_QP        (VMCI_LOCK_RANK_QPHEADER - 1)
//#define VMCI_LOCK_RANK_PACKET_QP        0xffd /* For vVol */

#define VMCI_SEMA_RANK_QUEUEPAIRLIST    (VMCI_SEMA_RANK_QPHEADER - 1)
#define VMCI_SEMA_RANK_GUESTMEM         (VMCI_SEMA_RANK_QUEUEPAIRLIST - 1)

/*
 * Host specific struct used for signalling.
 */

typedef struct VMCIHost {
#if defined(VMKERNEL)
   World_ID vmmWorldID[2];   /*
                              * First one is the active one and the second
                              * one is shadow world during FSR.
                              */
#elif defined(__linux__)
   wait_queue_head_t  waitQueue;
#elif defined(__APPLE__)
   struct Socket *socket; /* vmci Socket object on Mac OS. */
#elif defined(_WIN32)
   KEVENT *callEvent; /* Ptr to userlevel event used when signalling
                       * new pending guestcalls in kernel.
                       */
#endif
} VMCIHost;

/*
 * Guest device port I/O.
 */

#if defined(__linux__)
   typedef unsigned short int VMCIIoPort;
   typedef int VMCIIoHandle;
#elif defined(_WIN32)
   typedef PUCHAR VMCIIoPort;
   typedef int VMCIIoHandle;
#elif defined(__APPLE__)
   typedef unsigned short int VMCIIoPort;
   typedef void *VMCIIoHandle;
#endif // __APPLE__

void VMCI_ReadPortBytes(VMCIIoHandle handle, VMCIIoPort port, uint8 *buffer,
                        size_t bufferLength);

int VMCI_InitLock(VMCILock *lock, char *name, VMCILockRank rank);
void VMCI_CleanupLock(VMCILock *lock);
void VMCI_GrabLock(VMCILock *lock, VMCILockFlags *flags);
void VMCI_ReleaseLock(VMCILock *lock, VMCILockFlags flags);
void VMCI_GrabLock_BH(VMCILock *lock, VMCILockFlags *flags);
void VMCI_ReleaseLock_BH(VMCILock *lock, VMCILockFlags flags);

void VMCIHost_InitContext(VMCIHost *hostContext, uintptr_t eventHnd);
void VMCIHost_ReleaseContext(VMCIHost *hostContext);
void VMCIHost_SignalCall(VMCIHost *hostContext);
void VMCIHost_ClearCall(VMCIHost *hostContext);
Bool VMCIHost_WaitForCallLocked(VMCIHost *hostContext,
                                VMCILock *lock,
                                VMCILockFlags *flags,
                                Bool useBH);
#ifdef VMKERNEL
int VMCIHost_ContextToHostVmID(VMCIHost *hostContext, VMCIHostVmID *hostVmID);
int VMCIHost_ContextHasUuid(VMCIHost *hostContext, const char *uuid);
void VMCIHost_SetActiveHnd(VMCIHost *hostContext, uintptr_t eventHnd);
Bool VMCIHost_RemoveHnd(VMCIHost *hostContext, uintptr_t eventHnd);
Bool VMCIHost_IsActiveHnd(VMCIHost *hostContext, uintptr_t eventHnd);
void VMCIHost_SetInactiveHnd(VMCIHost *hostContext, uintptr_t eventHnd);
uint32 VMCIHost_NumHnds(VMCIHost *hostContext);
uintptr_t VMCIHost_GetActiveHnd(VMCIHost *hostContext);
void VMCIHost_SignalBitmap(VMCIHost *hostContext);
void VMCIHost_SignalBitmapAlways(VMCIHost *hostContext);
void VMCIHost_SignalCallAlways(VMCIHost *hostContext);
#endif

#if defined(_WIN32)
   /*
    * On Windows, Driver Verifier will panic() if we leak memory when we are
    * unloaded.  It dumps the leaked blocks for us along with callsites, which
    * it handily tracks, but if we embed ExAllocate() inside a function, then
    * the callsite is useless.  So make this a macro on this platform only.
    */
#  define VMCI_AllocKernelMem(_sz, _f)                       \
      ExAllocatePoolWithTag((((_f) & VMCI_MEMORY_NONPAGED) ? \
                             NonPagedPool : PagedPool),      \
                            (_sz), 'MMTC')
#else // _WIN32
void *VMCI_AllocKernelMem(size_t size, int flags);
#endif // _WIN32
void VMCI_FreeKernelMem(void *ptr, size_t size);

int VMCI_CopyToUser(VA64 dst, const void *src, size_t len);
Bool VMCIWellKnownID_AllowMap(VMCIId wellKnownID,
                              VMCIPrivilegeFlags privFlags);

int VMCIHost_CompareUser(VMCIHostUser *user1, VMCIHostUser *user2);

void VMCI_CreateEvent(VMCIEvent *event);
void VMCI_DestroyEvent(VMCIEvent *event);
void VMCI_SignalEvent(VMCIEvent *event);
void VMCI_WaitOnEvent(VMCIEvent *event, VMCIEventReleaseCB releaseCB,
		      void *clientData);
#if (defined(__APPLE__) || defined(__linux__) || defined(_WIN32)) && !defined(VMKERNEL)
Bool VMCI_WaitOnEventInterruptible(VMCIEvent *event,
                                   VMCIEventReleaseCB releaseCB,
                                   void *clientData);
#endif

#if !defined(VMKERNEL) && (defined(__linux__) || defined(_WIN32) || \
                           defined(__APPLE__))
int VMCI_CopyFromUser(void *dst, VA64 src, size_t len);
#endif

typedef void (VMCIWorkFn)(void *data);
Bool VMCI_CanScheduleDelayedWork(void);
int VMCI_ScheduleDelayedWork(VMCIWorkFn *workFn, void *data);

int VMCIMutex_Init(VMCIMutex *mutex, char *name, VMCILockRank rank);
void VMCIMutex_Destroy(VMCIMutex *mutex);
void VMCIMutex_Acquire(VMCIMutex *mutex);
void VMCIMutex_Release(VMCIMutex *mutex);

#if defined(_WIN32) || defined(__APPLE__)
int VMCIKernelIf_Init(void);
void VMCIKernelIf_Exit(void);
#if defined(_WIN32)
void VMCIKernelIf_DrainDelayedWork(void);
#endif // _WIN32
#endif // _WIN32 || __APPLE__

#if !defined(VMKERNEL) && \
    (defined(__linux__) || defined(_WIN32) || defined(__APPLE__))
void *VMCI_AllocQueue(uint64 size, uint32 flags);
void VMCI_FreeQueue(void *q, uint64 size);
typedef struct PPNSet {
  uint64      numProducePages;
  uint64      numConsumePages;
  VMCIPpnList producePPNs;
  VMCIPpnList consumePPNs;
  Bool        initialized;
} PPNSet;
int VMCI_AllocPPNSet(void *produceQ, uint64 numProducePages, void *consumeQ,
                     uint64 numConsumePages, PPNSet *ppnSet);
void VMCI_FreePPNSet(PPNSet *ppnSet);
int VMCI_PopulatePPNList(uint8 *callBuf, const PPNSet *ppnSet);
#endif

struct VMCIQueue;

struct PageStoreAttachInfo;
struct VMCIQueue *VMCIHost_AllocQueue(uint64 queueSize);
void VMCIHost_FreeQueue(struct VMCIQueue *queue, uint64 queueSize);

#if defined(VMKERNEL)
typedef World_Handle *VMCIGuestMemID;
#define INVALID_VMCI_GUEST_MEM_ID  NULL
#else
typedef uint32 VMCIGuestMemID;
#define INVALID_VMCI_GUEST_MEM_ID  0
#endif

#if defined(VMKERNEL) || defined(__linux__)  || defined(_WIN32) || \
    defined(__APPLE__)
  struct QueuePairPageStore;
  int VMCIHost_RegisterUserMemory(struct QueuePairPageStore *pageStore,
                                  struct VMCIQueue *produceQ,
                                  struct VMCIQueue *consumeQ);
  void VMCIHost_UnregisterUserMemory(struct VMCIQueue *produceQ,
                                     struct VMCIQueue *consumeQ);
  int VMCIHost_MapQueues(struct VMCIQueue *produceQ,
                         struct VMCIQueue *consumeQ,
                         uint32 flags);
  int VMCIHost_UnmapQueues(VMCIGuestMemID gid,
                           struct VMCIQueue *produceQ,
                           struct VMCIQueue *consumeQ);
  void VMCI_InitQueueMutex(struct VMCIQueue *produceQ,
                           struct VMCIQueue *consumeQ);
  void VMCI_CleanupQueueMutex(struct VMCIQueue *produceQ,
                              struct VMCIQueue *consumeQ);
  int VMCI_AcquireQueueMutex(struct VMCIQueue *queue, Bool canBlock);
  void VMCI_ReleaseQueueMutex(struct VMCIQueue *queue);
#else // Below are the guest OS'es without host side support.
#  define VMCI_InitQueueMutex(_pq, _cq)
#  define VMCI_CleanupQueueMutex(_pq, _cq) do { } while (0)
#  define VMCI_AcquireQueueMutex(_q, _cb) VMCI_SUCCESS
#  define VMCI_ReleaseQueueMutex(_q) do { } while (0)
#  define VMCIHost_RegisterUserMemory(_ps, _pq, _cq) VMCI_ERROR_UNAVAILABLE
#  define VMCIHost_UnregisterUserMemory(_pq, _cq) do { } while (0)
#  define VMCIHost_MapQueues(_pq, _cq, _f) VMCI_SUCCESS
#  define VMCIHost_UnmapQueues(_gid, _pq, _cq) VMCI_SUCCESS
#endif

#if defined(VMKERNEL)
  void VMCIHost_MarkQueuesAvailable(struct VMCIQueue *produceQ,
                                    struct VMCIQueue *consumeQ);
  void VMCIHost_MarkQueuesUnavailable(struct VMCIQueue *produceQ,
                                      struct VMCIQueue *consumeQ);
  int VMCIHost_RevalidateQueues(struct VMCIQueue *produceQ,
                                struct VMCIQueue *consumeQ);
#else
#  define VMCIHost_MarkQueuesAvailable(_q, _p) do { } while (0)
#  define VMCIHost_MarkQueuesUnavailable(_q, _p) do { } while(0)
#endif

#if defined(VMKERNEL) || defined(__linux__)
   void VMCI_LockQueueHeader(struct VMCIQueue *queue);
   void VMCI_UnlockQueueHeader(struct VMCIQueue *queue);
#else
#  define VMCI_LockQueueHeader(_q) NOT_IMPLEMENTED()
#  define VMCI_UnlockQueueHeader(_q) NOT_IMPLEMENTED()
#endif

#if defined(VMKERNEL)
   int VMCI_QueueHeaderUpdated(struct VMCIQueue *produceQ);
#else
#  define VMCI_QueueHeaderUpdated(_q) VMCI_SUCCESS
#endif

#if (!defined(VMKERNEL) && defined(__linux__)) || defined(_WIN32) ||  \
    defined(__APPLE__)
  int VMCIHost_GetUserMemory(VA64 produceUVA, VA64 consumeUVA,
                             struct VMCIQueue *produceQ,
                             struct VMCIQueue *consumeQ);
  void VMCIHost_ReleaseUserMemory(struct VMCIQueue *produceQ,
                                  struct VMCIQueue *consumeQ);
#else
#  define VMCIHost_GetUserMemory(_puva, _cuva, _pq, _cq) VMCI_ERROR_UNAVAILABLE
#  define VMCIHost_ReleaseUserMemory(_pq, _cq) NOT_IMPLEMENTED()
#endif

#if defined(_WIN32)
    Bool VMCI_EnqueueToDevNull(struct VMCIQueue *queue);
    int VMCI_ConvertToLocalQueue(struct VMCIQueue *queueInfo,
                                 struct VMCIQueue *otherQueueInfo,
                                 uint64 size, Bool keepContent,
                                 void **oldQueue);
    void VMCI_RevertToNonLocalQueue(struct VMCIQueue *queueInfo,
                                    void *nonLocalQueue, uint64 size);
    void VMCI_FreeQueueBuffer(void *queue, uint64 size);
    Bool VMCI_CanCreate(void);
#else // _WIN32
#  define VMCI_EnqueueToDevNull(_q) FALSE
#  define VMCI_ConvertToLocalQueue(_pq, _cq, _s, _oq, _kc) VMCI_ERROR_UNAVAILABLE
#  define VMCI_RevertToNonLocalQueue(_q, _nlq, _s)
#  define VMCI_FreeQueueBuffer(_q, _s)
#  define VMCI_CanCreate() TRUE
#endif // !_WIN32
Bool VMCI_GuestPersonalityActive(void);
Bool VMCI_HostPersonalityActive(void);

#if defined(VMKERNEL)
  typedef List_Links VMCIListItem;
  typedef List_Links VMCIList;

#  define VMCIList_Init(_l)   List_Init(_l)
#  define VMCIList_InitEntry(_e)  List_InitElement(_e)
#  define VMCIList_Empty(_l)   List_IsEmpty(_l)
#  define VMCIList_Insert(_e, _l) List_Insert(_e, LIST_ATREAR(_l))
#  define VMCIList_Remove(_e) List_Remove(_e)
#  define VMCIList_Scan(_cur, _l) LIST_FORALL(_l, _cur)
#  define VMCIList_ScanSafe(_cur, _next, _l) LIST_FORALL_SAFE(_l, _cur, _next)
#  define VMCIList_Entry(_elem, _type, _field) List_Entry(_elem, _type, _field)
#  define VMCIList_First(_l) (VMCIList_Empty(_l)?NULL:List_First(_l))
#else
  typedef DblLnkLst_Links VMCIListItem;
  typedef DblLnkLst_Links VMCIList;

#  define VMCIList_Init(_l)   DblLnkLst_Init(_l)
#  define VMCIList_InitEntry(_e)   DblLnkLst_Init(_e)
#  define VMCIList_Empty(_l)   (!DblLnkLst_IsLinked(_l))
#  define VMCIList_Insert(_e, _l) DblLnkLst_LinkLast(_l, _e)
#  define VMCIList_Remove(_e) DblLnkLst_Unlink1(_e)
#  define VMCIList_Scan(_cur, _l) DblLnkLst_ForEach(_cur, _l)
#  define VMCIList_ScanSafe(_cur, _next, _l) DblLnkLst_ForEachSafe(_cur, _next, _l)
#  define VMCIList_Entry(_elem, _type, _field) DblLnkLst_Container(_elem, _type, _field)
#  define VMCIList_First(_l) (VMCIList_Empty(_l)?NULL:(_l)->next)
#endif

#if defined __cplusplus
} // extern "C"
#endif

#endif // _VMCI_KERNEL_IF_H_
���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vmware_pack_init.h��������������������������������������������������������������0000444�0000000�0000000�00000003651�13432725350�017107� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __VMWARE_PACK_INIT_H__
#   define __VMWARE_PACK_INIT_H__

/*
 * vmware_pack_init.h --
 *
 *    Platform-independent code to make the compiler pack (i.e. have them
 *    occupy the smallest possible space) structure definitions. The following
 *    constructs are known to work --hpreg
 *
 *    #include "vmware_pack_begin.h"
 *    struct foo {
 *       ...
 *    }
 *    #include "vmware_pack_end.h"
 *    ;
 *
 *    typedef
 *    #include "vmware_pack_begin.h"
 *    struct foo {
 *       ...
 *    }
 *    #include "vmware_pack_end.h"
 *    foo;
 */

#ifdef _MSC_VER
/*
 * MSVC 6.0 emits warning 4103 when the pack push and pop pragma pairing is
 * not balanced within 1 included file. That is annoying because our scheme
 * is based on the pairing being balanced between 2 included files.
 *
 * So we disable this warning, but this is safe because the compiler will also
 * emit warning 4161 when there is more pops than pushes within 1 main
 * file --hpreg
 */

#   pragma warning(disable:4103)
#elif __GNUC__
#else
#   error Compiler packing...
#endif

#endif /* __VMWARE_PACK_INIT_H__ */
���������������������������������������������������������������������������������������vmxnet3-only/shared/vm_basic_defs.h�����������������������������������������������������������������0000444�0000000�0000000�00000044517�13432725350�016357� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vm_basic_defs.h --
 *
 *	Standard macros for VMware source code.
 */

#ifndef _VM_BASIC_DEFS_H_
#define _VM_BASIC_DEFS_H_

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_VMKDRIVERS
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMCORE
#include "includeCheck.h"
#include "vm_basic_types.h" // For INLINE.

/* Checks for FreeBSD, filtering out VMKERNEL. */
#if !defined(VMKERNEL) && defined(__FreeBSD__)
#define __IS_FREEBSD__ 1
#else
#define __IS_FREEBSD__ 0
#endif
#define __IS_FREEBSD_VER__(ver) (__IS_FREEBSD__ && __FreeBSD_version >= (ver))

#if defined _WIN32 && defined USERLEVEL
   #include <stddef.h>  /*
                         * We redefine offsetof macro from stddef; make
                         * sure that it's already defined before we do that.
                         */
   #include <windows.h>	// for Sleep() and LOWORD() etc.
   #undef GetFreeSpace  // Unpollute preprocessor namespace.
#endif

/*
 * Simple macros
 */

#ifndef vmw_offsetof
#define vmw_offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#endif

#if (defined __APPLE__ || defined __FreeBSD__) && \
    (!defined KERNEL && !defined _KERNEL && !defined VMKERNEL && !defined __KERNEL__)
#   include <stddef.h>
#else
#ifndef offsetof
#define VMW_DEFINED_OFFSETOF

/*
 * XXX While the _WIN32 implementation appears to be identical to vmw_offsetof
 * in terms of behavior, they need to be separate to match verbatim the
 * definition used by the respective compilers, to avoid a redefinition warning.
 *
 * This is necessary until we eliminate the inclusion of <windows.h> above.
 */
#ifdef _WIN32
#define offsetof(s,m)   (size_t)&(((s *)0)->m)
/*
 * We use the builtin offset for gcc/clang, except when we're running under the
 * vmkernel's GDB macro preprocessor, since gdb doesn't understand
 * __builtin_offsetof.
 */
#elif defined __GNUC__ && !defined VMKERNEL_GDB_MACRO_BUILDER
#define offsetof __builtin_offsetof
#else
#define offsetof vmw_offsetof
#endif

#endif // offsetof
#endif // __APPLE__

#define VMW_CONTAINER_OF(ptr, type, member) \
   ((type *)((char *)(ptr) - vmw_offsetof(type, member)))

#ifndef ARRAYSIZE
#define ARRAYSIZE(a) (sizeof (a) / sizeof *(a))
#endif

#ifndef MIN
#define MIN(_a, _b)   (((_a) < (_b)) ? (_a) : (_b))
#endif

/* The Solaris 9 cross-compiler complains about these not being used */
#ifndef sun
static INLINE int
Min(int a, int b)
{
   return a < b ? a : b;
}
#endif

#ifndef MAX
#define MAX(_a, _b)   (((_a) > (_b)) ? (_a) : (_b))
#endif

#ifndef sun
static INLINE int
Max(int a, int b)
{
   return a > b ? a : b;
}
#endif

#define VMW_CLAMP(x, min, max) \
   ((x) < (min) ? (min) : ((x) > (max) ? (max) : (x)))

#define ROUNDUP(x,y)		(((x) + (y) - 1) / (y) * (y))
#define ROUNDDOWN(x,y)		((x) / (y) * (y))
#define ROUNDUPBITS(x, bits)	(((uintptr_t) (x) + MASK(bits)) & ~MASK(bits))
#define ROUNDDOWNBITS(x, bits)	((uintptr_t) (x) & ~MASK(bits))
#define CEILING(x, y)		(((x) + (y) - 1) / (y))
#if defined __APPLE__
#include <machine/param.h>
#undef MASK
#endif

/*
 * The MASK macro behaves badly when given negative numbers or numbers larger
 * than the highest order bit number (e.g. 32 on a 32-bit machine) as an
 * argument. The range 0..31 is safe.
 */

#define MASK(n)		((1 << (n)) - 1)	    /* make an n-bit mask */
#define MASK64(n)	((CONST64U(1) << (n)) - 1)  /* make an n-bit mask */
/*
 * MASKRANGE64 makes a bit vector starting at bit lo and ending at bit hi.  No
 * checking for lo < hi is done.
 */
#define MASKRANGE64(hi, lo)      (MASK64((hi) - (lo) + 1) << (lo))

/* SIGNEXT64 sign extends a n-bit value to 64-bits. */
#define SIGNEXT64(val, n)       (((int64)(val) << (64 - (n))) >> (64 - (n)))

#define DWORD_ALIGN(x)          ((((x) + 3) >> 2) << 2)
#define QWORD_ALIGN(x)          ((((x) + 7) >> 3) << 3)

#define IMPLIES(a,b) (!(a) || (b))

/*
 * Not everybody (e.g., the monitor) has NULL
 */

#ifndef NULL
#ifdef  __cplusplus
#define NULL    0
#else
#define NULL    ((void *)0)
#endif
#endif

/*
 * Token concatenation
 *
 * The C preprocessor doesn't prescan arguments when they are
 * concatenated or stringified.  So we need extra levels of
 * indirection to convince the preprocessor to expand its
 * arguments.
 */

#define CONC(x, y)              x##y
#define XCONC(x, y)             CONC(x, y)
#define XXCONC(x, y)            XCONC(x, y)
#define MAKESTR(x)              #x
#define XSTR(x)                 MAKESTR(x)

/*
 * Wide versions of string constants.
 */

#ifndef WSTR
#define WSTR_(X)     L ## X
#define WSTR(X)      WSTR_(X)
#endif

/*
 * Page operations
 *
 * It has been suggested that these definitions belong elsewhere
 * (like x86types.h).  However, I deem them common enough
 * (since even regular user-level programs may want to do
 * page-based memory manipulation) to be here.
 * -- edward
 */

#ifndef PAGE_SHIFT // {
#if defined VM_I386
   #define PAGE_SHIFT    12
#elif defined __APPLE__
   #define PAGE_SHIFT    12
#elif defined VM_ARM_64
   #define PAGE_SHIFT    12
#elif defined __arm__
   #define PAGE_SHIFT    12
#else
   #error
#endif
#endif // }

#ifndef PAGE_SIZE
#define PAGE_SIZE     (1 << PAGE_SHIFT)
#endif

#ifndef PAGE_MASK
#define PAGE_MASK     (PAGE_SIZE - 1)
#endif

#ifndef PAGE_OFFSET
#define PAGE_OFFSET(_addr)  ((uintptr_t)(_addr) & (PAGE_SIZE - 1))
#endif

#ifndef PAGE_NUMBER
#define PAGE_NUMBER(_addr)  ((uintptr_t)(_addr) / PAGE_SIZE)
#endif

#ifndef VM_PAGE_BASE
#define VM_PAGE_BASE(_addr)  ((_addr) & ~(PAGE_SIZE - 1))
#endif

#ifndef VM_PAGES_SPANNED
#define VM_PAGES_SPANNED(_addr, _size) \
   ((((_addr) & (PAGE_SIZE - 1)) + (_size) + (PAGE_SIZE - 1)) >> PAGE_SHIFT)
#endif

#ifndef BYTES_2_PAGES
#define BYTES_2_PAGES(_nbytes)  ((_nbytes) >> PAGE_SHIFT)
#endif

#ifndef PAGES_2_BYTES
#define PAGES_2_BYTES(_npages)  (((uint64)(_npages)) << PAGE_SHIFT)
#endif

#ifndef MBYTES_SHIFT
#define MBYTES_SHIFT 20
#endif

#ifndef MBYTES_2_PAGES
#define MBYTES_2_PAGES(_nbytes) \
   ((uint64)(_nbytes) << (MBYTES_SHIFT - PAGE_SHIFT))
#endif

#ifndef PAGES_2_MBYTES
#define PAGES_2_MBYTES(_npages) ((_npages) >> (MBYTES_SHIFT - PAGE_SHIFT))
#endif

#ifndef ROUNDUP_PAGES_2_MBYTES
#define ROUNDUP_PAGES_2_MBYTES(_npages) \
(((_npages) + MASK(MBYTES_SHIFT - PAGE_SHIFT)) >> (MBYTES_SHIFT - PAGE_SHIFT))
#endif

#ifndef ROUNDDOWN_PAGES_2_MBYTES
#define ROUNDDOWN_PAGES_2_MBYTES(_npages) \
((_npages) >> (MBYTES_SHIFT - PAGE_SHIFT))
#endif

#ifndef GBYTES_2_PAGES
#define GBYTES_2_PAGES(_nbytes) ((uint64)(_nbytes) << (30 - PAGE_SHIFT))
#endif

#ifndef PAGES_2_GBYTES
#define PAGES_2_GBYTES(_npages) ((_npages) >> (30 - PAGE_SHIFT))
#endif

#ifndef BYTES_2_MBYTES
#define BYTES_2_MBYTES(_nbytes) ((_nbytes) >> MBYTES_SHIFT)
#endif

#ifndef MBYTES_2_BYTES
#define MBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << MBYTES_SHIFT)
#endif

#ifndef BYTES_2_GBYTES
#define BYTES_2_GBYTES(_nbytes) ((_nbytes) >> 30)
#endif

#ifndef GBYTES_2_BYTES
#define GBYTES_2_BYTES(_nbytes) ((uint64)(_nbytes) << 30)
#endif

#ifndef VM_PAE_LARGE_PAGE_SHIFT
#define VM_PAE_LARGE_PAGE_SHIFT 21
#endif

#ifndef VM_PAE_LARGE_PAGE_SIZE
#define VM_PAE_LARGE_PAGE_SIZE (1 << VM_PAE_LARGE_PAGE_SHIFT)
#endif

#ifndef VM_PAE_LARGE_PAGE_MASK
#define VM_PAE_LARGE_PAGE_MASK (VM_PAE_LARGE_PAGE_SIZE - 1)
#endif

#ifndef VM_PAE_LARGE_2_SMALL_PAGES
#define VM_PAE_LARGE_2_SMALL_PAGES (BYTES_2_PAGES(VM_PAE_LARGE_PAGE_SIZE))
#endif

#ifndef VM_PAE_LARGE_2_BYTES
#define VM_PAE_LARGE_2_BYTES(_2mbytes) ((_2mbytes) << VM_PAE_LARGE_PAGE_SHIFT)
#endif

#ifndef VM_1GB_PAGE_SHIFT
#define VM_1GB_PAGE_SHIFT 30
#endif

#ifndef VM_1GB_PAGE_SIZE
#define VM_1GB_PAGE_SIZE (1 << VM_1GB_PAGE_SHIFT)
#endif

#ifndef VM_1GB_2_PAGES
#define VM_1GB_2_PAGES (BYTES_2_PAGES(VM_1GB_PAGE_SIZE))
#endif

#ifndef VM_1GB_2_PDIRS
#define VM_1GB_2_PDIRS (VM_1GB_PAGE_SIZE / VM_PAE_LARGE_PAGE_SIZE)
#endif

/*
 * Word operations
 */

#ifndef LOWORD
#define LOWORD(_dw)   ((_dw) & 0xffff)
#endif
#ifndef HIWORD
#define HIWORD(_dw)   (((_dw) >> 16) & 0xffff)
#endif

#ifndef LOBYTE
#define LOBYTE(_w)    ((_w) & 0xff)
#endif
#ifndef HIBYTE
#define HIBYTE(_w)    (((_w) >> 8) & 0xff)
#endif

#ifndef HIDWORD
#define HIDWORD(_qw)   ((uint32)((_qw) >> 32))
#endif
#ifndef LODWORD
#define LODWORD(_qw)   ((uint32)(_qw))
#endif
#define QWORD(_hi, _lo)   ((((uint64)(_hi)) << 32) | ((uint32)(_lo)))

/*
 * Deposit a field _src at _pos bits from the right,
 * with a length of _len, into the integer _target.
 */

#define DEPOSIT_BITS(_src,_pos,_len,_target) { \
	unsigned mask = ((1 << _len) - 1); \
	unsigned shiftedmask = ((1 << _len) - 1) << _pos; \
	_target = (_target & ~shiftedmask) | ((_src & mask) << _pos); \
}

/*
 * Get return address.
 */

#ifdef _MSC_VER
#ifdef __cplusplus
extern "C"
#endif
void *_ReturnAddress(void);
#pragma intrinsic(_ReturnAddress)
#define GetReturnAddress() _ReturnAddress()
#elif __GNUC__
#define GetReturnAddress() __builtin_return_address(0)
#endif

#ifdef __GNUC__
#ifndef sun

/*
 * A bug in __builtin_frame_address was discovered in gcc 4.1.1, and
 * fixed in 4.2.0; assume it originated in 4.0. PR 147638 and 554369.
 */
#if  !(__GNUC__ == 4 && (__GNUC_MINOR__ == 0 || __GNUC_MINOR__ == 1))
#define GetFrameAddr() __builtin_frame_address(0)
#endif

#endif // sun
#endif // __GNUC__

/*
 * Data prefetch was added in gcc 3.1.1
 * http://www.gnu.org/software/gcc/gcc-3.1/changes.html
 */
#ifdef __GNUC__
#  if ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ > 1) || \
       (__GNUC__ == 3 && __GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 1))
#     define PREFETCH_R(var) __builtin_prefetch((var), 0 /* read */, \
                                                3 /* high temporal locality */)
#     define PREFETCH_W(var) __builtin_prefetch((var), 1 /* write */, \
                                                3 /* high temporal locality */)
#  else
#     define PREFETCH_R(var) ((void)(var))
#     define PREFETCH_W(var) ((void)(var))
#  endif
#endif /* __GNUC__ */

#ifdef USERLEVEL // {

/*
 * Standardize some Posix names on Windows.
 */

#ifdef _WIN32 // {

/* Conflict with definition of Visual Studio 2015 */
#if (_MSC_VER < 1900)
#define snprintf  _snprintf
#endif

#define strtok_r  strtok_s

#if (_MSC_VER < 1500)
#define	vsnprintf _vsnprintf
#endif

typedef int uid_t;
typedef int gid_t;

static INLINE void
sleep(unsigned int sec)
{
   Sleep(sec * 1000);
}

static INLINE int
usleep(unsigned long usec)
{
   Sleep(CEILING(usec, 1000));

return 0;
}

typedef int pid_t;
#define       F_OK          0
#define       X_OK          1
#define       W_OK          2
#define       R_OK          4

#endif // }

/*
 * Macro for username comparison.
 */

#ifdef _WIN32 // {
#define USERCMP(x,y)  Str_Strcasecmp(x,y)
#else
#define USERCMP(x,y)  strcmp(x,y)
#endif // }

#endif // }

#ifndef va_copy

#ifdef _WIN32

/*
 * Windows needs va_copy. This works for both 32 and 64-bit Windows
 * based on inspection of how varags.h from the Visual C CRTL is
 * implemented. (Future versions of the RTL may break this).
 */

#define va_copy(dest, src) ((dest) = (src))

#elif defined(__APPLE__) && defined(KERNEL)

// The macOS kernel SDK defines va_copy in stdarg.h.
#include <stdarg.h>

#elif defined(__GNUC__) && (__GNUC__ < 3)

/*
 * Old versions of gcc recognize __va_copy, but not va_copy.
 */

#define va_copy(dest, src) __va_copy(dest, src)

#endif // _WIN32

#endif // va_copy

/*
 * This one is outside USERLEVEL because it's used by
 * files compiled into the Windows hgfs driver or the display
 * driver.
 */

#if defined(_WIN32) && defined(_MSC_VER)
#define PATH_MAX 256
#ifndef strcasecmp
#define strcasecmp(_s1,_s2)   _stricmp((_s1),(_s2))
#endif
#ifndef strncasecmp
#define strncasecmp(_s1,_s2,_n)   _strnicmp((_s1),(_s2),(_n))
#endif
#endif

/*
 * Convenience definitions of unicode characters.
 */

#ifndef UTF8_ELLIPSIS
#define UTF8_ELLIPSIS "\xe2\x80\xa6"
#endif

/*
 * Convenience macros and definitions. Can often be used instead of #ifdef.
 */

#undef ARM64_ONLY
#ifdef VM_ARM_64
#define ARM64_ONLY(x)    x
#else
#define ARM64_ONLY(x)
#endif

#undef X86_ONLY
#ifdef VM_X86_ANY
#define X86_ONLY(x)      x
#else
#define X86_ONLY(x)
#endif

#undef DEBUG_ONLY
#ifdef VMX86_DEBUG
#define vmx86_debug      1
#define DEBUG_ONLY(x)    x
#else
#define vmx86_debug      0
#define DEBUG_ONLY(x)
#endif

#ifdef VMX86_STATS
#define vmx86_stats   1
#define STATS_ONLY(x) x
#else
#define vmx86_stats   0
#define STATS_ONLY(x)
#endif

#ifdef VMX86_DEVEL
#define vmx86_devel   1
#define DEVEL_ONLY(x) x
#else
#define vmx86_devel   0
#define DEVEL_ONLY(x)
#endif

#ifdef VMX86_LOG
#define vmx86_log     1
#define LOG_ONLY(x)   x
#else
#define vmx86_log     0
#define LOG_ONLY(x)
#endif

#ifdef VMX86_BETA
#define vmx86_beta     1
#define BETA_ONLY(x)   x
#else
#define vmx86_beta     0
#define BETA_ONLY(x)
#endif

#ifdef VMX86_RELEASE
#define vmx86_release   1
#define RELEASE_ONLY(x) x
#else
#define vmx86_release   0
#define RELEASE_ONLY(x)
#endif

#ifdef VMX86_SERVER
#define vmx86_server 1
#define SERVER_ONLY(x) x
#define HOSTED_ONLY(x)
#else
#define vmx86_server 0
#define SERVER_ONLY(x)
#define HOSTED_ONLY(x) x
#endif

#ifdef VMKERNEL
#define vmkernel 1
#define VMKERNEL_ONLY(x) x
#else
#define vmkernel 0
#define VMKERNEL_ONLY(x)
#endif

#ifdef _WIN32
#define WIN32_ONLY(x) x
#define POSIX_ONLY(x)
#define vmx86_win32 1
#else
#define WIN32_ONLY(x)
#define POSIX_ONLY(x) x
#define vmx86_win32 0
#endif

#ifdef __linux__
#define vmx86_linux 1
#define LINUX_ONLY(x) x
#else
#define vmx86_linux 0
#define LINUX_ONLY(x)
#endif

#ifdef __APPLE__
#define vmx86_apple 1
#define APPLE_ONLY(x) x
#else
#define vmx86_apple 0
#define APPLE_ONLY(x)
#endif

#if defined(__APPLE__) && defined(VMW_APPLE_SANDBOX)
#define vmw_apple_sandbox 1
#else
#define vmw_apple_sandbox 0
#endif

#if defined(__APPLE__) && defined(VMW_APPLE_APP_STORE)
#define vmw_apple_app_store 1
#else
#define vmw_apple_app_store 0
#endif

#ifdef VMM
#define VMM_ONLY(x) x
#else
#define VMM_ONLY(x)
#endif

#if defined(VMM) || defined(VMKERNEL)
#define USER_ONLY(x)
#else
#define USER_ONLY(x) x
#endif

#ifdef _WIN32
#define VMW_INVALID_HANDLE INVALID_HANDLE_VALUE
#else
#define VMW_INVALID_HANDLE (-1LL)
#endif

#ifdef _WIN32
#define fsync(fd) _commit(fd)
#define fileno(f) _fileno(f)
#else
#endif

/*
 * Debug output macros for Windows drivers (the Eng variant is for
 * display/printer drivers only.
 */
#ifdef _WIN32
#ifndef USES_OLD_WINDDK
#if defined(VMX86_LOG)
#ifdef _WIN64
#define WinDrvPrint(arg, ...) DbgPrintEx(DPFLTR_IHVDRIVER_ID, (ULONG)~0, arg, __VA_ARGS__)
#else
#define WinDrvPrint(arg, ...) DbgPrint(arg, __VA_ARGS__)
#endif
#define WinDrvEngPrint(arg, ...) EngDbgPrint(arg, __VA_ARGS__)
#else
#define WinDrvPrint(arg, ...)
#define WinDrvEngPrint(arg, ...)
#endif
#endif
#endif // _WIN32

#ifdef HOSTED_LG_PG
#define hosted_lg_pg 1
#else
#define hosted_lg_pg 0
#endif

/*
 * Use to initialize cbSize for this structure to preserve < Vista
 * compatibility.
 */
#define NONCLIENTMETRICSINFO_V1_SIZE CCSIZEOF_STRUCT(NONCLIENTMETRICS, \
                                                     lfMessageFont)

/* This is not intended to be thread-safe. */
#define DO_ONCE(code)                                                   \
   do {                                                                 \
      static Bool _doOnceDone = FALSE;                                  \
      if (UNLIKELY(!_doOnceDone)) {                                     \
         _doOnceDone = TRUE;                                            \
         code;                                                          \
      }                                                                 \
   } while (0)

/*
 * Bug 827422 and 838523.
 */

#if defined __GNUC__ && __GNUC__ >= 4
#define VISIBILITY_HIDDEN __attribute__((visibility("hidden")))
#else
#define VISIBILITY_HIDDEN /* nothing */
#endif

/*
 * Bitfield extraction.
 */

#define EXTRACT_BITSLICE32(_val , _lsb, _msb)  \
   (((uint32)(_val) << (31 - (_msb))) >> ((31 - (_msb)) + (_lsb)))
#define EXTRACT_BITFIELD32(_val, _pos, _len) \
   EXTRACT_BITSLICE32((_val), (_pos), ((_pos) + (_len) - 1))
#define EXTRACT_BITSLICE64(_val, _lsb, _msb) \
   (((uint64)(_val) << (63 - (_msb))) >> ((63 - (_msb)) + (_lsb)))
#define EXTRACT_BITFIELD64(_val, _pos, _len) \
   EXTRACT_BITSLICE64((_val), (_pos), ((_pos) + (_len) - 1))

/*
 * Typical cache line size.  Use this for aligning structures to cache
 * lines for performance, but do not rely on it for correctness.
 *
 * On x86, all current processors newer than P4 have 64-byte lines,
 * but P4 had 128.
 *
 * On ARM, the line size can vary between cores.  64-byte lines are
 * common, but either larger or smaller powers of two are possible.
 */
#define CACHELINE_SIZE             64
#define CACHELINE_SHIFT            6
#define CACHELINE_ALIGNMENT_MASK   (CACHELINE_SIZE - 1)

/*
 * Bits to bytes sizes.
 */

#define SIZE_8BIT   1
#define SIZE_16BIT  2
#define SIZE_24BIT  3
#define SIZE_32BIT  4
#define SIZE_48BIT  6
#define SIZE_64BIT  8
#define SIZE_80BIT  10
#define SIZE_128BIT 16
#define SIZE_256BIT 32
#define SIZE_512BIT 64

/*
 * Allocate a variable of type _type, aligned to _align bytes, returning a
 * pointer to the variable in _var.  Potentially _align - 1 bytes may be
 * wasted.  On x86, GCC 6.3.0 behaves sub-optimally when variables are declared
 * on the stack using the aligned attribute, so this pattern is preferred.
 * See PRs 1795155, 1819963.
 */
#define WITH_PTR_TO_ALIGNED_VAR(_type, _align, _var)                     \
   do {                                                                  \
      uint8 _buf_##_var[sizeof(_type) + (_align) - 1];                   \
      _type *_var = (_type *) ((uintptr_t)(_buf_##_var + (_align) - 1) & \
                               ~((uintptr_t) ((_align) - 1)));

#define END_PTR_TO_ALIGNED_VAR \
   } while (0)

#endif // ifndef _VM_BASIC_DEFS_H_
���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/backdoor_def.h������������������������������������������������������������������0000444�0000000�0000000�00000033730�13432725350�016170� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * backdoor_def.h --
 *
 * This contains backdoor defines that can be included from
 * an assembly language file.
 */

#ifndef _BACKDOOR_DEF_H_
#define _BACKDOOR_DEF_H_

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_VMKERNEL
#include "includeCheck.h"

#if defined __cplusplus
extern "C" {
#endif

/*
 * If you want to add a new low-level backdoor call for a guest userland
 * application, please consider using the GuestRpc mechanism instead. --hpreg
 */

#define BDOOR_MAGIC 0x564D5868

/* Low-bandwidth backdoor port. --hpreg */

#define BDOOR_PORT 0x5658

#define   BDOOR_CMD_GETMHZ                    1
/*
 * BDOOR_CMD_APMFUNCTION is used by:
 *
 * o The FrobOS code, which instead should either program the virtual chipset
 *   (like the new BIOS code does, matthias offered to implement that), or not
 *   use any VM-specific code (which requires that we correctly implement
 *   "power off on CLI HLT" for SMP VMs, boris offered to implement that)
 *
 * o The old BIOS code, which will soon be jettisoned
 *
 *  --hpreg
 */
#define   BDOOR_CMD_APMFUNCTION               2 /* CPL0 only. */
#define   BDOOR_CMD_GETDISKGEO                3
#define   BDOOR_CMD_GETPTRLOCATION            4
#define   BDOOR_CMD_SETPTRLOCATION            5
#define   BDOOR_CMD_GETSELLENGTH              6
#define   BDOOR_CMD_GETNEXTPIECE              7
#define   BDOOR_CMD_SETSELLENGTH              8
#define   BDOOR_CMD_SETNEXTPIECE              9
#define   BDOOR_CMD_GETVERSION               10
#define   BDOOR_CMD_GETDEVICELISTELEMENT     11
#define   BDOOR_CMD_TOGGLEDEVICE             12
#define   BDOOR_CMD_GETGUIOPTIONS            13
#define   BDOOR_CMD_SETGUIOPTIONS            14
#define   BDOOR_CMD_GETSCREENSIZE            15
#define   BDOOR_CMD_MONITOR_CONTROL          16 /* Disabled by default. */
#define   BDOOR_CMD_GETHWVERSION             17
#define   BDOOR_CMD_OSNOTFOUND               18 /* CPL0 only. */
#define   BDOOR_CMD_GETUUID                  19
#define   BDOOR_CMD_GETMEMSIZE               20
//#define BDOOR_CMD_HOSTCOPY                 21 /* Not in use. Was devel only. */
//#define BDOOR_CMD_SERVICE_VM               22 /* Not in use. Never shipped. */
#define   BDOOR_CMD_GETTIME                  23 /* Deprecated -> GETTIMEFULL. */
#define   BDOOR_CMD_STOPCATCHUP              24
#define   BDOOR_CMD_PUTCHR                   25 /* Disabled by default. */
#define   BDOOR_CMD_ENABLE_MSG               26 /* Devel only. */
//#define BDOOR_CMD_GOTO_TCL                 27 /* Not in use. Was devel only */
#define   BDOOR_CMD_INITPCIOPROM             28 /* CPL 0 only. */
//#define BDOOR_CMD_INT13                    29 /* Not in use. */
#define   BDOOR_CMD_MESSAGE                  30
#define   BDOOR_CMD_SIDT                     31
#define   BDOOR_CMD_SGDT                     32
#define   BDOOR_CMD_SLDT_STR                 33
#define   BDOOR_CMD_ISACPIDISABLED           34
//#define BDOOR_CMD_TOE                      35 /* Not in use. */
#define   BDOOR_CMD_ISMOUSEABSOLUTE          36
#define   BDOOR_CMD_PATCH_SMBIOS_STRUCTS     37 /* CPL 0 only. */
#define   BDOOR_CMD_MAPMEM                   38 /* Devel only */
#define   BDOOR_CMD_ABSPOINTER_DATA          39
#define   BDOOR_CMD_ABSPOINTER_STATUS        40
#define   BDOOR_CMD_ABSPOINTER_COMMAND       41
//#define BDOOR_CMD_TIMER_SPONGE             42 /* Not in use. */
#define   BDOOR_CMD_PATCH_ACPI_TABLES        43 /* CPL 0 only. */
//#define BDOOR_CMD_DEVEL_FAKEHARDWARE       44 /* Not in use. */
#define   BDOOR_CMD_GETHZ                    45
#define   BDOOR_CMD_GETTIMEFULL              46
//#define BDOOR_CMD_STATELOGGER              47 /* Not in use. */
#define   BDOOR_CMD_CHECKFORCEBIOSSETUP      48 /* CPL 0 only. */
#define   BDOOR_CMD_LAZYTIMEREMULATION       49 /* CPL 0 only. */
#define   BDOOR_CMD_BIOSBBS                  50 /* CPL 0 only. */
//#define BDOOR_CMD_VASSERT                  51 /* Not in use. */
#define   BDOOR_CMD_ISGOSDARWIN              52
#define   BDOOR_CMD_DEBUGEVENT               53
#define   BDOOR_CMD_OSNOTMACOSXSERVER        54 /* CPL 0 only. */
#define   BDOOR_CMD_GETTIMEFULL_WITH_LAG     55
#define   BDOOR_CMD_ACPI_HOTPLUG_DEVICE      56 /* Devel only. */
#define   BDOOR_CMD_ACPI_HOTPLUG_MEMORY      57 /* Devel only. */
#define   BDOOR_CMD_ACPI_HOTPLUG_CBRET       58 /* Devel only. */
//#define BDOOR_CMD_GET_HOST_VIDEO_MODES     59 /* Not in use. */
#define   BDOOR_CMD_ACPI_HOTPLUG_CPU         60 /* Devel only. */
//#define BDOOR_CMD_USB_HOTPLUG_MOUSE        61 /* Not in use. Never shipped. */
#define   BDOOR_CMD_XPMODE                   62 /* CPL 0 only. */
#define   BDOOR_CMD_NESTING_CONTROL          63
#define   BDOOR_CMD_FIRMWARE_INIT            64 /* CPL 0 only. */
#define   BDOOR_CMD_FIRMWARE_ACPI_SERVICES   65 /* CPL 0 only. */
#  define BDOOR_CMD_FAS_GET_TABLE_SIZE        0
#  define BDOOR_CMD_FAS_GET_TABLE_DATA        1
#  define BDOOR_CMD_FAS_GET_PLATFORM_NAME     2
#  define BDOOR_CMD_FAS_GET_PCIE_OSC_MASK     3
#  define BDOOR_CMD_FAS_GET_APIC_ROUTING      4
#  define BDOOR_CMD_FAS_GET_TABLE_SKIP        5
#  define BDOOR_CMD_FAS_GET_SLEEP_ENABLES     6
#  define BDOOR_CMD_FAS_GET_HARD_RESET_ENABLE 7
#  define BDOOR_CMD_FAS_GET_MOUSE_HID         8
#  define BDOOR_CMD_FAS_GET_SMBIOS_VERSION    9
#  define BDOOR_CMD_FAS_GET_64BIT_PCI_HOLE_SIZE 10
//#define BDOOR_CMD_FAS_GET_NVDIMM_FMT_CODE  11 /* Not in use. Never shipped. */
#  define BDOOR_CMD_FAS_SRP_ENABLED          12
#  define BDOOR_CMD_FAS_EXIT_BOOT_SERVICES   13
#define   BDOOR_CMD_SENDPSHAREHINTS          66 /* Not in use. Deprecated. */
#define   BDOOR_CMD_ENABLE_USB_MOUSE         67
#define   BDOOR_CMD_GET_VCPU_INFO            68
#  define BDOOR_CMD_VCPU_SLC64                0
#  define BDOOR_CMD_VCPU_SYNC_VTSCS           1
#  define BDOOR_CMD_VCPU_HV_REPLAY_OK         2
#  define BDOOR_CMD_VCPU_LEGACY_X2APIC_OK     3
#  define BDOOR_CMD_VCPU_MMIO_HONORS_PAT      4
#  define BDOOR_CMD_VCPU_RESERVED            31
#define   BDOOR_CMD_EFI_SERIALCON_CONFIG     69 /* CPL 0 only. */
#define   BDOOR_CMD_BUG328986                70 /* CPL 0 only. */
#define   BDOOR_CMD_FIRMWARE_ERROR           71 /* CPL 0 only. */
#  define BDOOR_CMD_FE_INSUFFICIENT_MEM       0
#  define BDOOR_CMD_FE_EXCEPTION              1
#  define BDOOR_CMD_FE_SGX                    2
#  define BDOOR_CMD_FE_PCI_MMIO               3
#define   BDOOR_CMD_VMK_INFO                 72
#define   BDOOR_CMD_EFI_BOOT_CONFIG          73 /* CPL 0 only. */
#  define BDOOR_CMD_EBC_LEGACYBOOT_ENABLED        0
#  define BDOOR_CMD_EBC_GET_ORDER                 1
#  define BDOOR_CMD_EBC_SHELL_ACTIVE              2
#  define BDOOR_CMD_EBC_GET_NETWORK_BOOT_PROTOCOL 3
#  define BDOOR_CMD_EBC_QUICKBOOT_ENABLED         4
#  define BDOOR_CMD_EBC_GET_PXE_ARCH              5
#define   BDOOR_CMD_GET_HW_MODEL             74 /* CPL 0 only. */
#define   BDOOR_CMD_GET_SVGA_CAPABILITIES    75 /* CPL 0 only. */
#define   BDOOR_CMD_GET_FORCE_X2APIC         76 /* CPL 0 only  */
#define   BDOOR_CMD_SET_PCI_HOLE             77 /* CPL 0 only  */
#define   BDOOR_CMD_GET_PCI_HOLE             78 /* CPL 0 only  */
#define   BDOOR_CMD_GET_PCI_BAR              79 /* CPL 0 only  */
#define   BDOOR_CMD_SHOULD_GENERATE_SYSTEMID 80 /* CPL 0 only  */
#define   BDOOR_CMD_READ_DEBUG_FILE          81 /* Devel only. */
#define   BDOOR_CMD_SCREENSHOT               82 /* Devel only. */
#define   BDOOR_CMD_INJECT_KEY               83 /* Devel only. */
#define   BDOOR_CMD_INJECT_MOUSE             84 /* Devel only. */
#define   BDOOR_CMD_MKS_GUEST_STATS          85 /* CPL 0 only. */
#  define BDOOR_CMD_MKSGS_RESET               0
#  define BDOOR_CMD_MKSGS_ADD_PPN             1
#  define BDOOR_CMD_MKSGS_REMOVE_PPN          2
#define   BDOOR_CMD_ABSPOINTER_RESTRICT      86
#define   BDOOR_CMD_GUEST_INTEGRITY          87
#  define BDOOR_CMD_GI_GET_CAPABILITIES       0
#  define BDOOR_CMD_GI_SETUP_ENTRY_POINT      1
#  define BDOOR_CMD_GI_SETUP_ALERTS           2
#  define BDOOR_CMD_GI_SETUP_STORE            3
#  define BDOOR_CMD_GI_SETUP_EVENT_RING       4
#  define BDOOR_CMD_GI_SETUP_NON_FAULT_READ   5
#  define BDOOR_CMD_GI_ENTER_INTEGRITY_MODE   6
#  define BDOOR_CMD_GI_EXIT_INTEGRITY_MODE    7
#  define BDOOR_CMD_GI_RESET_INTEGRITY_MODE   8
#  define BDOOR_CMD_GI_GET_EVENT_RING_STATE   9
#  define BDOOR_CMD_GI_CONSUME_RING_EVENTS   10
#  define BDOOR_CMD_GI_WATCH_MAPPINGS_START  11
#  define BDOOR_CMD_GI_WATCH_MAPPINGS_STOP   12
#  define BDOOR_CMD_GI_CHECK_MAPPINGS_NOW    13
#  define BDOOR_CMD_GI_WATCH_PPNS_START      14
#  define BDOOR_CMD_GI_WATCH_PPNS_STOP       15
#  define BDOOR_CMD_GI_SEND_MSG              16
#  define BDOOR_CMD_GI_TEST_READ_MOB        128
#  define BDOOR_CMD_GI_TEST_ADD_EVENT       129
#  define BDOOR_CMD_GI_TEST_MAPPING         130
#  define BDOOR_CMD_GI_TEST_PPN             131
#  define BDOOR_CMD_GI_MAX                  131
#define   BDOOR_CMD_MKSSTATS_SNAPSHOT        88 /* Devel only. */
#  define BDOOR_CMD_MKSSTATS_START            0
#  define BDOOR_CMD_MKSSTATS_STOP             1
#define   BDOOR_CMD_SECUREBOOT               89
#define   BDOOR_CMD_COPY_PHYSMEM             90 /* Devel only. */
#define   BDOOR_CMD_STEALCLOCK               91 /* CPL 0 only. */
#  define BDOOR_STEALCLOCK_STATUS_DISABLED    0
#  define BDOOR_STEALCLOCK_STATUS_ENABLED     1
#define   BDOOR_CMD_GUEST_PAGE_HINTS         92 /* CPL 0 only  */
#define   BDOOR_CMD_FIRMWARE_UPDATE          93 /* CPL 0 only. */
#  define BDOOR_CMD_FU_GET_HOST_VERSION       0
#  define BDOOR_CMD_FU_UPDATE_FROM_HOST       1
#  define BDOOR_CMD_FU_LOCK                   2
#define   BDOOR_CMD_FUZZER_HELPER            94 /* Devel only. */
#  define BDOOR_CMD_FUZZER_INIT               0
#  define BDOOR_CMD_FUZZER_NEXT               1
#define   BDOOR_CMD_PUTCHR12                 95
#define   BDOOR_CMD_MAX                      96

/*
 * IMPORTANT NOTE: When modifying the behavior of an existing backdoor command,
 * you must adhere to the semantics expected by the oldest Tools who use that
 * command. Specifically, do not alter the way in which the command modifies
 * the registers. Otherwise backwards compatibility will suffer.
 */

/* Nesting control operations */

#define NESTING_CONTROL_RESTRICT_BACKDOOR 0
#define NESTING_CONTROL_OPEN_BACKDOOR     1
#define NESTING_CONTROL_QUERY             2
#define NESTING_CONTROL_MAX               2

/* EFI Boot Order options, nibble-sized. */
#define EFI_BOOT_ORDER_TYPE_EFI           0x0
#define EFI_BOOT_ORDER_TYPE_LEGACY        0x1
#define EFI_BOOT_ORDER_TYPE_NONE          0xf

#define BDOOR_NETWORK_BOOT_PROTOCOL_NONE  0x0
#define BDOOR_NETWORK_BOOT_PROTOCOL_IPV4  0x1
#define BDOOR_NETWORK_BOOT_PROTOCOL_IPV6  0x2

#define BDOOR_SECUREBOOT_STATUS_DISABLED  0xFFFFFFFFUL
#define BDOOR_SECUREBOOT_STATUS_APPROVED  1
#define BDOOR_SECUREBOOT_STATUS_DENIED    2

/* High-bandwidth backdoor port. --hpreg */

#define BDOORHB_PORT 0x5659
#define BDOORHB_CMD_MESSAGE 0
#define BDOORHB_CMD_VASSERT 1
#define BDOORHB_CMD_MAX 2

/*
 * There is another backdoor which allows access to certain TSC-related
 * values using otherwise illegal PMC indices when the pseudo_perfctr
 * control flag is set.
 */

#define BDOOR_PMC_HW_TSC      0x10000
#define BDOOR_PMC_REAL_NS     0x10001
#define BDOOR_PMC_APPARENT_NS 0x10002
#define BDOOR_PMC_PSEUDO_TSC  0x10003

#define IS_BDOOR_PMC(index)  (((index) | 3) == 0x10003)
#define BDOOR_CMD(ecx)       ((ecx) & 0xffff)

/* Sub commands for BDOOR_CMD_VMK_INFO */
#define BDOOR_CMD_VMK_INFO_ENTRY   1

/*
 * Current format for the guest page hints is:
 *
 * Arg0: BDOOR_MAGIC, Arg3: BDOOR_PORT
 *
 * Arg1: (rbx on x86)
 *
 *  0         64
 *  |   PPN   |
 *
 * Arg2: (rcx on x86)
 *
 *  0         16        32         64
 *  | Command |  Type   | Reserved |
 *
 * Arg4: (rsi on x86)
 *
 *  0          16         64
 *  | numPages | Reserved |
 *
 */
#define BDOOR_GUEST_PAGE_HINTS_NOT_SUPPORTED ((unsigned)-1)
#define BDOOR_GUEST_PAGE_HINTS_MAX_PAGES     (0xffff)
#define BDOOR_GUEST_PAGE_HINTS_TYPE_PSHARE   (0)
#define BDOOR_GUEST_PAGE_HINTS_TYPE(reg)     (((reg) >> 16) & 0xffff)

#ifdef VMM
/*
 *----------------------------------------------------------------------
 *
 * Backdoor_CmdRequiresFullyValidVCPU --
 *
 *    A few backdoor commands require the full VCPU to be valid
 *    (including GDTR, IDTR, TR and LDTR). The rest get read/write
 *    access to GPRs and read access to Segment registers (selectors).
 *
 * Result:
 *    True iff VECX contains a command that require the full VCPU to
 *    be valid.
 *
 *----------------------------------------------------------------------
 */
static INLINE Bool
Backdoor_CmdRequiresFullyValidVCPU(unsigned cmd)
{
   return cmd == BDOOR_CMD_SIDT ||
          cmd == BDOOR_CMD_SGDT ||
          cmd == BDOOR_CMD_SLDT_STR;
}
#endif

#ifdef VM_ARM_64

#define BDOOR_ARM64_LB_PORT      (BDOOR_PORT)
#define BDOOR_ARM64_HB_PORT_IN   (BDOORHB_PORT)
#define BDOOR_ARM64_HB_PORT_OUT  (BDOORHB_PORT +1)

#define BDOOR_ARG0 REG_X0
#define BDOOR_ARG1 REG_X1
#define BDOOR_ARG2 REG_X2
#define BDOOR_ARG3 REG_X3
#define BDOOR_ARG4 REG_X4
#define BDOOR_ARG5 REG_X5
#define BDOOR_ARG6 REG_X6

#else

#define BDOOR_ARG0 REG_RAX
#define BDOOR_ARG1 REG_RBX
#define BDOOR_ARG2 REG_RCX
#define BDOOR_ARG3 REG_RDX
#define BDOOR_ARG4 REG_RSI
#define BDOOR_ARG5 REG_RDI
#define BDOOR_ARG6 REG_RBP

#endif

#if defined __cplusplus
}
#endif

#endif // _BACKDOOR_DEF_H_
����������������������������������������vmxnet3-only/shared/compat_ethtool.h����������������������������������������������������������������0000444�0000000�0000000�00000003662�13432725347�016616� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef _COMPAT_ETHTOOL_H
#define _COMPAT_ETHTOOL_H

/*
 * ethtool is a userspace utility for getting and setting ethernet device
 * settings. Kernel support for it was first published in 2.4.0-test11, but
 * only in 2.4.15 were the ethtool_value struct and the ETHTOOL_GLINK ioctl
 * added to ethtool.h (together, because the ETHTOOL_GLINK ioctl expects a 
 * single value response).
 *
 * Likewise, ioctls for getting and setting TSO were published in 2.4.22.
 */

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 0)
#   include <linux/ethtool.h>

#   ifndef ETHTOOL_GLINK
#      define ETHTOOL_GLINK 0x0a

typedef struct {
	__u32 cmd;
	__u32 data;
} compat_ethtool_value;

#   else

typedef struct ethtool_value compat_ethtool_value;
#   endif

#   ifndef ETHTOOL_GTSO
#      define ETHTOOL_GTSO 0x1E
#      define ETHTOOL_STSO 0x1F
#   endif
#endif

#if COMPAT_LINUX_VERSION_CHECK_LT(3, 3, 0)
#   define compat_ethtool_rxfh_indir_default(i, num_queues) (i % num_queues)
#else
#   define compat_ethtool_rxfh_indir_default(i, num_queues) ethtool_rxfh_indir_default(i, num_queues)
#endif

#endif /* _COMPAT_ETHTOOL_H */
������������������������������������������������������������������������������vmxnet3-only/shared/vm_basic_asm_x86_common.h�������������������������������������������������������0000444�0000000�0000000�00000033175�13432725350�020271� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2013-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vm_basic_asm_x86_common.h --
 *
 *	Basic assembler macros common to 32-bit and 64-bit x86 ISA.
 */

#ifndef _VM_BASIC_ASM_X86_COMMON_H_
#define _VM_BASIC_ASM_X86_COMMON_H_

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMCORE
#include "includeCheck.h"

#ifndef VM_X86_ANY
#error "Should be included only in x86 builds"
#endif

/*
 * x86-64 windows doesn't support inline asm so we have to use these
 * intrinsic functions defined in the compiler.  Not all of these are well
 * documented.  There is an array in the compiler dll (c1.dll) which has
 * an array of the names of all the intrinsics minus the leading
 * underscore.  Searching around in the ntddk.h file can also be helpful.
 *
 * The declarations for the intrinsic functions were taken from the DDK.
 * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
 * will complain about second linkage of the intrinsic functions.
 * We define the intrinsic using the basic types corresponding to the
 * Windows typedefs. This avoids having to include windows header files
 * to get to the windows types.
 */
#if defined(_MSC_VER) && !defined(BORA_NO_WIN32_INTRINS)
#ifdef __cplusplus
extern "C" {
#endif
/*
 * It seems x86 & x86-64 windows still implements these intrinsic
 * functions.  The documentation for the x86-64 suggest the
 * __inbyte/__outbyte intrinsics even though the _in/_out work fine and
 * __inbyte/__outbyte aren't supported on x86.
 */
int            _inp(unsigned short);
unsigned short _inpw(unsigned short);
unsigned long  _inpd(unsigned short);

int            _outp(unsigned short, int);
unsigned short _outpw(unsigned short, unsigned short);
unsigned long  _outpd(uint16, unsigned long);
#pragma intrinsic(_inp, _inpw, _inpd, _outp, _outpw, _outpw, _outpd)

/*
 * Prevents compiler from re-ordering reads, writes and reads&writes.
 * These functions do not add any instructions thus only affect
 * the compiler ordering.
 *
 * See:
 * `Lockless Programming Considerations for Xbox 360 and Microsoft Windows'
 * http://msdn.microsoft.com/en-us/library/bb310595(VS.85).aspx
 */
void _ReadBarrier(void);
void _WriteBarrier(void);
void _ReadWriteBarrier(void);
#pragma intrinsic(_ReadBarrier, _WriteBarrier, _ReadWriteBarrier)

void _mm_mfence(void);
void _mm_lfence(void);
#pragma intrinsic(_mm_mfence, _mm_lfence)

long _InterlockedXor(long volatile *, long);
#pragma intrinsic(_InterlockedXor)

unsigned int __getcallerseflags(void);
#pragma intrinsic(__getcallerseflags)

#ifdef VM_X86_64
/*
 * intrinsic functions only supported by x86-64 windows as of 2k3sp1
 */
unsigned __int64 __rdtsc(void);
void             __stosw(unsigned short *, unsigned short, size_t);
void             __stosd(unsigned long *, unsigned long, size_t);
void             _mm_pause(void);
#pragma intrinsic(__rdtsc, __stosw, __stosd, _mm_pause)

unsigned char  _BitScanForward64(unsigned long *, unsigned __int64);
unsigned char  _BitScanReverse64(unsigned long *, unsigned __int64);
#pragma intrinsic(_BitScanForward64, _BitScanReverse64)
#endif /* VM_X86_64 */

unsigned char  _BitScanForward(unsigned long *, unsigned long);
unsigned char  _BitScanReverse(unsigned long *, unsigned long);
#pragma intrinsic(_BitScanForward, _BitScanReverse)

unsigned char  _bittest(const long *, long);
unsigned char  _bittestandset(long *, long);
unsigned char  _bittestandreset(long *, long);
unsigned char  _bittestandcomplement(long *, long);
#pragma intrinsic(_bittest, _bittestandset, _bittestandreset, _bittestandcomplement)
#ifdef VM_X86_64
unsigned char  _bittestandset64(__int64 *, __int64);
unsigned char  _bittestandreset64(__int64 *, __int64);
#pragma intrinsic(_bittestandset64, _bittestandreset64)
#endif // VM_X86_64
#ifdef __cplusplus
}
#endif
#endif // _MSC_VER

#ifdef __GNUC__
/*
 * Checked against the Intel manual and GCC --hpreg
 *
 * volatile because reading from port can modify the state of the underlying
 * hardware.
 *
 * Note: The undocumented %z construct doesn't work (internal compiler error)
 *       with gcc-2.95.1
 */

#define __GCC_IN(s, type, name) \
static INLINE type              \
name(uint16 port)               \
{                               \
   type val;                    \
                                \
   __asm__ __volatile__(        \
      "in" #s " %w1, %0"        \
      : "=a" (val)              \
      : "Nd" (port)             \
   );                           \
                                \
   return val;                  \
}

__GCC_IN(b, uint8, INB)
__GCC_IN(w, uint16, INW)
__GCC_IN(l, uint32, IN32)

/*
 * Checked against the Intel manual and GCC --hpreg
 *
 * Note: The undocumented %z construct doesn't work (internal compiler error)
 *       with gcc-2.95.1
 */

#define __GCC_OUT(s, s2, port, val) do { \
   __asm__(                              \
      "out" #s " %" #s2 "1, %w0"         \
      :                                  \
      : "Nd" (port), "a" (val)           \
   );                                    \
} while (0)

#define OUTB(port, val) __GCC_OUT(b, b, port, val)
#define OUTW(port, val) __GCC_OUT(w, w, port, val)
#define OUT32(port, val) __GCC_OUT(l, , port, val)

#define GET_CURRENT_EIP(_eip) \
      __asm__ __volatile("call 0\n\tpopl %0" : "=r" (_eip): );

static INLINE unsigned int
GetCallerEFlags(void)
{
   unsigned long flags;
   asm volatile("pushf; pop %0" : "=r"(flags));
   return flags;
}

#elif defined(_MSC_VER)
static INLINE  uint8
INB(uint16 port)
{
   return (uint8)_inp(port);
}
static INLINE void
OUTB(uint16 port, uint8 value)
{
   _outp(port, value);
}
static INLINE uint16
INW(uint16 port)
{
   return _inpw(port);
}
static INLINE void
OUTW(uint16 port, uint16 value)
{
   _outpw(port, value);
}
static INLINE  uint32
IN32(uint16 port)
{
   return _inpd(port);
}
static INLINE void
OUT32(uint16 port, uint32 value)
{
   _outpd(port, value);
}

#ifndef VM_X86_64
#ifdef NEAR
#undef NEAR
#endif

#define GET_CURRENT_EIP(_eip) do { \
   __asm call NEAR PTR $+5 \
   __asm pop eax \
   __asm mov _eip, eax \
} while (0)
#endif // VM_X86_64

static INLINE unsigned int
GetCallerEFlags(void)
{
   return __getcallerseflags();
}

#endif // __GNUC__

/* Sequence recommended by Intel for the Pentium 4. */
#define INTEL_MICROCODE_VERSION() (             \
   __SET_MSR(MSR_BIOS_SIGN_ID, 0),              \
   __GET_EAX_FROM_CPUID(1),                     \
   __GET_MSR(MSR_BIOS_SIGN_ID))

/*
 *-----------------------------------------------------------------------------
 *
 * RDTSC_BARRIER --
 *
 *      Implements an RDTSC fence.  Instructions executed prior to the
 *      fence will have completed before the fence and all stores to
 *      memory are flushed from the store buffer.
 *
 *      On AMD, MFENCE is sufficient.  On Intel, only LFENCE is
 *      documented to fence RDTSC, but LFENCE won't drain the store
 *      buffer.  So, use MFENCE;LFENCE, which will work on both AMD and
 *      Intel.
 *
 *      It is the callers' responsibility to check for SSE2 before
 *      calling this function.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      Cause loads and stores prior to this to be globally visible, and
 *      RDTSC will not pass.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
RDTSC_BARRIER(void)
{
#ifdef __GNUC__
   __asm__ __volatile__(
      "mfence \n\t"
      "lfence \n\t"
      ::: "memory"
   );
#elif defined _MSC_VER
   /* Prevent compiler from moving code across mfence/lfence. */
   _ReadWriteBarrier();
   _mm_mfence();
   _mm_lfence();
   _ReadWriteBarrier();
#else
#error No compiler defined for RDTSC_BARRIER
#endif
}

/*
 * Memory Barriers
 * ===============
 *
 *    Terminology
 *    -----------
 *
 * A compiler memory barrier prevents the compiler from re-ordering memory
 * accesses accross the barrier. It is not a CPU instruction, it is a compiler
 * directive (i.e. it does not emit any code).
 *
 * A CPU memory barrier prevents the CPU from re-ordering memory accesses
 * accross the barrier. It is a CPU instruction.
 *
 * A memory barrier is the union of a compiler memory barrier and a CPU memory
 * barrier. A compiler memory barrier is a useless construct by itself. It is
 * only useful when combined with a CPU memory barrier, to implement a memory
 * barrier.
 *
 *    Semantics
 *    ---------
 *
 * At the time COMPILER_*_BARRIER were created (and references to them were
 * added to the code), the code was only targetting x86. The intent of the code
 * was really to use a memory barrier, but because x86 uses a strongly ordered
 * memory model, the CPU would not re-order memory accesses, and the code could
 * get away with using just a compiler memory barrier. So COMPILER_*_BARRIER
 * were born and were implemented as compiler memory barriers _on x86_. But
 * make no mistake, _the semantics that the code expects from
 * COMPILER_*_BARRIER are that of a memory barrier_!
 *
 *    DO NOT USE!
 *    -----------
 *
 * On at least one non-x86 architecture, COMPILER_*_BARRIER are
 * 1) Misnomers
 * 2) Not fine-grained enough to provide the best performance.
 * For the above two reasons, usage of COMPILER_*_BARRIER is now deprecated.
 * _Do not add new references to COMPILER_*_BARRIER._ Instead, precisely
 * document the intent of your code by using
 * <mem_type/purpose>_<before_access_type>_BARRIER_<after_access_type>.
 * Existing references to COMPILER_*_BARRIER are being slowly but surely
 * converted, and when no references are left, COMPILER_*_BARRIER will be
 * retired.
 *
 * Thanks for pasting this whole comment into every architecture header.
 */

#if defined __GNUC__
#   define COMPILER_READ_BARRIER()  COMPILER_MEM_BARRIER()
#   define COMPILER_WRITE_BARRIER() COMPILER_MEM_BARRIER()
#   define COMPILER_MEM_BARRIER()   __asm__ __volatile__("" ::: "memory")
#elif defined _MSC_VER
#   define COMPILER_READ_BARRIER()  _ReadBarrier()
#   define COMPILER_WRITE_BARRIER() _WriteBarrier()
#   define COMPILER_MEM_BARRIER()   _ReadWriteBarrier()
#endif

/*
 * Memory barriers. These take the form of
 *
 * <mem_type/purpose>_<before_access_type>_BARRIER_<after_access_type>
 *
 * where:
 *   <mem_type/purpose> is either SMP, DMA, or MMIO.
 *   <*_access type> is either R(load), W(store) or RW(any).
 *
 * Above every use of these memory barriers in the code, there _must_ be a
 * comment to justify the use, i.e. a comment which:
 *
 * 1) Precisely identifies which memory accesses must not be re-ordered across
 *    the memory barrier.
 * 2) Explains why it is important that the memory accesses not be re-ordered.
 *
 * Thanks for pasting this whole comment into every architecture header.
 *
 * On x86, we only need to care specifically about store-load reordering on
 * normal memory types. In other cases, only a compiler barrier is needed.
 * SMP_W_BARRIER_R is implemented with a locked xor operation (instead of the
 * mfence instruction) for performance reasons. See PR 1674199 for more
 * details.
 *
 * On x64, special instructions are only provided for load-load (lfence) and
 * store-store (sfence) ordering, and they don't apply to normal memory.
 */

static INLINE void
SMP_W_BARRIER_R(void)
{
   volatile long temp;

#if defined __GNUC__
   __asm__ __volatile__ (
      "lock xorl $1, %0"
      : "+m" (temp)
      : /* no additional inputs */
      : "cc", "memory");
#elif defined _MSC_VER
   _InterlockedXor(&temp, 1);
#else
#error SMP_W_BARRIER_R not defined for this compiler
#endif
}

#define SMP_R_BARRIER_R()     COMPILER_READ_BARRIER()
#define SMP_R_BARRIER_W()     COMPILER_MEM_BARRIER()
#define SMP_R_BARRIER_RW()    COMPILER_MEM_BARRIER()
#define SMP_W_BARRIER_W()     COMPILER_WRITE_BARRIER()
#define SMP_W_BARRIER_RW()    SMP_W_BARRIER_R()
#define SMP_RW_BARRIER_R()    SMP_W_BARRIER_R()
#define SMP_RW_BARRIER_W()    COMPILER_MEM_BARRIER()
#define SMP_RW_BARRIER_RW()   SMP_W_BARRIER_R()

/*
 * Like the above, only for use with observers other than CPUs,
 * i.e. DMA masters.
 */

#define DMA_R_BARRIER_R()     SMP_R_BARRIER_R()
#define DMA_R_BARRIER_W()     SMP_R_BARRIER_W()
#define DMA_R_BARRIER_RW()    SMP_R_BARRIER_RW()
#define DMA_W_BARRIER_R()     SMP_W_BARRIER_R()
#define DMA_W_BARRIER_W()     SMP_W_BARRIER_W()
#define DMA_W_BARRIER_RW()    SMP_W_BARRIER_RW()
#define DMA_RW_BARRIER_R()    SMP_RW_BARRIER_R()
#define DMA_RW_BARRIER_W()    SMP_RW_BARRIER_W()
#define DMA_RW_BARRIER_RW()   SMP_RW_BARRIER_RW()

/*
 * And finally a set for use with MMIO accesses.
 */

#define MMIO_R_BARRIER_R()    SMP_R_BARRIER_R()
#define MMIO_R_BARRIER_W()    SMP_R_BARRIER_W()
#define MMIO_R_BARRIER_RW()   SMP_R_BARRIER_RW()
#define MMIO_W_BARRIER_R()    SMP_W_BARRIER_R()
#define MMIO_W_BARRIER_W()    SMP_W_BARRIER_W()
#define MMIO_W_BARRIER_RW()   SMP_W_BARRIER_RW()
#define MMIO_RW_BARRIER_R()   SMP_RW_BARRIER_R()
#define MMIO_RW_BARRIER_W()   SMP_RW_BARRIER_W()
#define MMIO_RW_BARRIER_RW()  SMP_RW_BARRIER_RW()

#endif // _VM_BASIC_ASM_X86_COMMON_H_
���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vm_device_version.h�������������������������������������������������������������0000444�0000000�0000000�00000034705�13432725350�017277� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998,2005-2012,2014-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef VM_DEVICE_VERSION_H
#define VM_DEVICE_VERSION_H

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMKDRIVERS
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_VMCORE
#include "includeCheck.h"

#ifdef _WIN32
#ifdef __MINGW32__
#include "initguid.h"
#else
#include "guiddef.h"
#endif
#endif

/* LSILogic 53C1030 Parallel SCSI controller
 * LSILogic SAS1068 SAS controller
 */
#define PCI_VENDOR_ID_LSILOGIC          0x1000
#define PCI_DEVICE_ID_LSI53C1030        0x0030
#define PCI_DEVICE_ID_LSISAS1068        0x0054

/* Our own PCI IDs
 *    VMware SVGA II (Unified VGA)
 *    VMware SVGA (PCI Accelerator)
 *    VMware vmxnet (Idealized NIC)
 *    VMware vmxscsi (Abortive idealized SCSI controller)
 *    VMware chipset (Subsystem ID for our motherboards)
 *    VMware e1000 (Subsystem ID)
 *    VMware vmxnet3 (Uniform Pass Through NIC)
 *    VMware HD Audio codec
 *    VMware HD Audio controller
 */
#define PCI_VENDOR_ID_VMWARE                    0x15AD
#define PCI_DEVICE_ID_VMWARE_SVGA2              0x0405
#define PCI_DEVICE_ID_VMWARE_SVGA               0x0710
#define PCI_DEVICE_ID_VMWARE_VGA                0x0711
#define PCI_DEVICE_ID_VMWARE_NET                0x0720
#define PCI_DEVICE_ID_VMWARE_SCSI               0x0730
#define PCI_DEVICE_ID_VMWARE_VMCI               0x0740
#define PCI_DEVICE_ID_VMWARE_CHIPSET            0x1976
#define PCI_DEVICE_ID_VMWARE_82545EM            0x0750 /* single port */
#define PCI_DEVICE_ID_VMWARE_82546EB            0x0760 /* dual port   */
#define PCI_DEVICE_ID_VMWARE_EHCI               0x0770
#define PCI_DEVICE_ID_VMWARE_UHCI               0x0774
#define PCI_DEVICE_ID_VMWARE_XHCI_0096          0x0778
#define PCI_DEVICE_ID_VMWARE_XHCI_0100          0x0779
#define PCI_DEVICE_ID_VMWARE_1394               0x0780
#define PCI_DEVICE_ID_VMWARE_BRIDGE             0x0790
#define PCI_DEVICE_ID_VMWARE_ROOTPORT           0x07A0
#define PCI_DEVICE_ID_VMWARE_VMXNET3            0x07B0
#define PCI_DEVICE_ID_VMWARE_PVSCSI             0x07C0
#define PCI_DEVICE_ID_VMWARE_82574              0x07D0
#define PCI_DEVICE_ID_VMWARE_AHCI               0x07E0
#define PCI_DEVICE_ID_VMWARE_NVME               0x07F0
#define PCI_DEVICE_ID_VMWARE_HDAUDIO_CODEC      0x1975
#define PCI_DEVICE_ID_VMWARE_HDAUDIO_CONTROLLER 0x1977

/*
 * TXT vendor, device and revision ID. We are keeping vendor
 * as Intel since tboot code does not like anything other
 * than Intel in the SINIT ACM header.
 */
#define TXT_VENDOR_ID                           0x8086
#define TXT_DEVICE_ID                           0xB002
#define TXT_REVISION_ID                         0x01

/* The hypervisor device might grow.  Please leave room
 * for 7 more subfunctions.
 */
#define PCI_DEVICE_ID_VMWARE_HYPER      0x0800
#define PCI_DEVICE_ID_VMWARE_VMI        0x0801

#define PCI_DEVICE_VMI_CLASS            0x05
#define PCI_DEVICE_VMI_SUBCLASS         0x80
#define PCI_DEVICE_VMI_INTERFACE        0x00
#define PCI_DEVICE_VMI_REVISION         0x01

/*
 * Device IDs for the PCI passthru test device:
 *
 * 0x0809 is for old fashioned PCI with MSI.
 * 0x080A is for PCI express with MSI-X.
 * 0x080B is for PCI express with configurable BARs.
 */
#define PCI_DEVICE_ID_VMWARE_PCI_TEST   0x0809
#define PCI_DEVICE_ID_VMWARE_PCIE_TEST1 0x080A
#define PCI_DEVICE_ID_VMWARE_PCIE_TEST2 0x080B

#define PCI_DEVICE_ID_VMWARE_VRDMA      0x0820
#define PCI_DEVICE_ID_VMWARE_VTPM       0x0830

/*
 * VMware Virtual Device Test Infrastructure (VDTI) devices
 */
#define PCI_DEVICE_ID_VMWARE_VDTI               0x7E57  /* stands for "TEST" */

/* From linux/pci_ids.h:
 *   AMD Lance Ethernet controller
 *   BusLogic SCSI controller
 *   Ensoniq ES1371 sound controller
 */
#define PCI_VENDOR_ID_AMD               0x1022
#define PCI_DEVICE_ID_AMD_VLANCE        0x2000
#define PCI_DEVICE_ID_AMD_IOMMU         0x1577
#define PCI_VENDOR_ID_BUSLOGIC			0x104B
#define PCI_DEVICE_ID_BUSLOGIC_MULTIMASTER_NC	0x0140
#define PCI_DEVICE_ID_BUSLOGIC_MULTIMASTER	0x1040
#define PCI_VENDOR_ID_ENSONIQ           0x1274
#define PCI_DEVICE_ID_ENSONIQ_ES1371    0x1371

/* From linux/pci_ids.h:
 *    Intel 82439TX (430 HX North Bridge)
 *    Intel 82371AB (PIIX4 South Bridge)
 *    Intel 82443BX (440 BX North Bridge and AGP Bridge)
 *    Intel 82545EM (e1000, server adapter, single port)
 *    Intel 82546EB (e1000, server adapter, dual port)
 *    Intel HECI (as embedded in ich9m)
 *    Intel XHCI (Panther Point / Intel 7 Series)
 */
#define PCI_VENDOR_ID_INTEL             0x8086
#define PCI_DEVICE_ID_INTEL_82439TX     0x7100
#define PCI_DEVICE_ID_INTEL_82371AB_0   0x7110
#define PCI_DEVICE_ID_INTEL_82371AB_2   0x7112
#define PCI_DEVICE_ID_INTEL_82371AB_3   0x7113
#define PCI_DEVICE_ID_INTEL_82371AB     0x7111
#define PCI_DEVICE_ID_INTEL_82443BX     0x7190
#define PCI_DEVICE_ID_INTEL_82443BX_1   0x7191
#define PCI_DEVICE_ID_INTEL_82443BX_2   0x7192 /* Used when no AGP support */
#define PCI_DEVICE_ID_INTEL_82545EM     0x100f
#define PCI_DEVICE_ID_INTEL_82546EB     0x1010
#define PCI_DEVICE_ID_INTEL_82574       0x10d3
#define PCI_DEVICE_ID_INTEL_82574_APPLE 0x10f6
#define PCI_DEVICE_ID_INTEL_HECI        0x2a74
#define PCI_DEVICE_ID_INTEL_PANTHERPOINT_XHCI 0x1e31

/*
 *  From drivers/usb/host/xhci-pci.c:
 *    Intel XHCI (Lynx Point / Intel 8 Series)
 */
#define PCI_DEVICE_ID_INTEL_LYNXPOINT_XHCI 0x8c31

/*
 * Intel Volume Management Device (VMD)
 */
#define PCI_DEVICE_ID_INTEL_VMD_V1           0x201d

/*
 * Intel Quickassist (QAT) devices.
 */
#define PCI_DEVICE_ID_INTEL_QAT_DH895XCC     0x0435
#define PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF  0x0443

#define PCI_DEVICE_ID_INTEL_QAT_C62X         0x37c8
#define PCI_DEVICE_ID_INTEL_QAT_C62X_VF      0x37c9

/*
 * Intel FPGAs
 */

#define PCI_DEVICE_ID_INTEL_FPGA_SKL_PF 0xbcc0
#define PCI_DEVICE_ID_INTEL_FPGA_SKL_VF 0xbcc1

#define E1000E_PCI_DEVICE_ID_CONFIG_STR "e1000e.pci.deviceID"
#define E1000E_PCI_SUB_VENDOR_ID_CONFIG_STR "e1000e.pci.subVendorID"
#define E1000E_PCI_SUB_DEVICE_ID_CONFIG_STR "e1000e.pci.subDeviceID"

/*
 * Intel HD Audio controller and Realtek ALC885 codec.
 */
#define PCI_DEVICE_ID_INTEL_631XESB_632XESB  0x269a
#define PCI_VENDOR_ID_REALTEK                0x10ec
#define PCI_DEVICE_ID_REALTEK_ALC885         0x0885

/*
 * Fresco Logic xHCI (USB 3.0) Controller
 */
#define PCI_VENDOR_ID_FRESCO            0x1B73
#define PCI_DEVICE_ID_FRESCO_FL1000     0x1000   // Original 1-port chip
#define PCI_DEVICE_ID_FRESCO_FL1009     0x1009   // New 2-port chip (Driver 3.0.98+)
#define PCI_DEVICE_ID_FRESCO_FL1400     0x1400   // Unknown (4-port? Dev hardware?)

/*
 * NEC/Renesas xHCI (USB 3.0) Controller
 */
#define PCI_VENDOR_ID_NEC               0x1033
#define PCI_DEVICE_ID_NEC_UPD720200     0x0194
#define PCI_REVISION_NEC_UPD720200      0x03
#define PCI_FIRMWARE_NEC_UPD720200      0x3015

#define SATA_ID_SERIAL_STR "00000000000000000001"  /* Must be 20 Bytes */
#define SATA_ID_FIRMWARE_STR  "00000001"    /* Must be 8 Bytes */

#define AHCI_ATA_MODEL_STR PRODUCT_GENERIC_NAME " Virtual SATA Hard Drive"
#define AHCI_ATAPI_MODEL_STR PRODUCT_GENERIC_NAME " Virtual SATA CDRW Drive"

/************* Strings for IDE Identity Fields **************************/
#define VIDE_ID_SERIAL_STR	"00000000000000000001"	/* Must be 20 Bytes */
#define VIDE_ID_FIRMWARE_STR	"00000001"		/* Must be 8 Bytes */

/* No longer than 40 Bytes */
#define VIDE_ATA_MODEL_STR PRODUCT_GENERIC_NAME " Virtual IDE Hard Drive"
#define VIDE_ATAPI_MODEL_STR PRODUCT_GENERIC_NAME " Virtual IDE CDROM Drive"

#define ATAPI_VENDOR_ID	"NECVMWar"		/* Must be 8 Bytes */
#define ATAPI_PRODUCT_ID PRODUCT_GENERIC_NAME " IDE CDROM"	/* Must be 16 Bytes */
#define ATAPI_REV_LEVEL	"1.00"			/* Must be 4 Bytes */

#define IDE_NUM_INTERFACES   2	/* support for two interfaces */
#define IDE_DRIVES_PER_IF    2

/************* Strings for SCSI Identity Fields **************************/
#define SCSI_DISK_MODEL_STR PRODUCT_GENERIC_NAME " Virtual SCSI Hard Drive"
#define SCSI_DISK_VENDOR_NAME COMPANY_NAME
#define SCSI_DISK_REV_LEVEL "1.0"
#define SCSI_CDROM_MODEL_STR PRODUCT_GENERIC_NAME " Virtual SCSI CDROM Drive"
#define SCSI_CDROM_VENDOR_NAME COMPANY_NAME
#define SCSI_CDROM_REV_LEVEL "1.0"

/************* NVME implementation limits ********************************/
#define NVME_MAX_CONTROLLERS   4
#define NVME_MIN_NAMESPACES    1
#define NVME_MAX_NAMESPACES    15 /* We support only 15 namespaces same
                                   * as SCSI devices.
                                   */

/************* SCSI implementation limits ********************************/
#define SCSI_MAX_CONTROLLERS	 4	  // Need more than 1 for MSCS clustering
#define	SCSI_MAX_DEVICES         16	  // BT-958 emulates only 16
#define PVSCSI_HWV14_MAX_DEVICES 65	  /* HWv14 And Later Supports 64 
					   * + controller at ID 7 
					   */
#define PVSCSI_MAX_DEVICES       255	  // 255 (including the controller)
#define PVSCSI_MAX_NUM_DISKS     (PVSCSI_HWV14_MAX_DEVICES - 1)

/************* SATA implementation limits ********************************/
#define SATA_MAX_CONTROLLERS   4
#define SATA_MAX_DEVICES       30
#define AHCI_MIN_PORTS         1
#define AHCI_MAX_PORTS SATA_MAX_DEVICES

/*
 * Publicly supported maximum number of disks per VM.
 */
#define MAX_NUM_DISKS \
   ((SATA_MAX_CONTROLLERS * SATA_MAX_DEVICES) + \
    (SCSI_MAX_CONTROLLERS * SCSI_MAX_DEVICES) + \
    (NVME_MAX_CONTROLLERS * NVME_MAX_NAMESPACES) + \
    (IDE_NUM_INTERFACES * IDE_DRIVES_PER_IF))

/*
 * Maximum number of supported disks in a VM from HWV14 or later, using PVSCSI updated max
 * devices.  The note above still holds true, but instead of publicly supporting
 * all devices, HWv14 simply extends the maximum support to 256 devices,
 * instead ~244 calculated above.
 *
 * PVSCSI_HW_MAX_DEVICES is 65 - allowing 64 disks + controller (at ID 7)
 * 4 * 64 = 256 devices.
 *
 */
#define MAX_NUM_DISKS_HWV14 MAX(MAX_NUM_DISKS, \
   (SCSI_MAX_CONTROLLERS * PVSCSI_MAX_NUM_DISKS))

/*
 * VSCSI_BV_INTS is the number of uint32's needed for a bit vector
 * to cover all scsi devices per target.
 */
#define VSCSI_BV_INTS            CEILING(PVSCSI_MAX_DEVICES, 8 * sizeof (uint32))
#define SCSI_IDE_CHANNEL         SCSI_MAX_CONTROLLERS
#define SCSI_IDE_HOSTED_CHANNEL  (SCSI_MAX_CONTROLLERS + 1)
#define SCSI_SATA_CHANNEL_FIRST  (SCSI_IDE_HOSTED_CHANNEL + 1)
#define SCSI_NVME_CHANNEL_FIRST  (SCSI_SATA_CHANNEL_FIRST + \
                                  SATA_MAX_CONTROLLERS)
#define SCSI_MAX_CHANNELS        (SCSI_NVME_CHANNEL_FIRST + \
                                  NVME_MAX_CONTROLLERS)

/************* SCSI-NVME channel IDs *******************************/
#define NVME_ID_TO_SCSI_ID(nvmeId)    \
   (SCSI_NVME_CHANNEL_FIRST + (nvmeId))

#define SCSI_ID_TO_NVME_ID(scsiId)    \
   ((scsiId) - SCSI_NVME_CHANNEL_FIRST)

/************* SCSI-SATA channel IDs********************************/
#define SATA_ID_TO_SCSI_ID(sataId)    \
   (SCSI_SATA_CHANNEL_FIRST + (sataId))

#define SCSI_ID_TO_SATA_ID(scsiId)    \
   ((scsiId) - SCSI_SATA_CHANNEL_FIRST)

/************* Strings for the VESA BIOS Identity Fields *****************/
#define VBE_OEM_STRING COMPANY_NAME " SVGA"
#define VBE_VENDOR_NAME COMPANY_NAME
#define VBE_PRODUCT_NAME PRODUCT_GENERIC_NAME

/************* PCI implementation limits ********************************/
#define PCI_MAX_BRIDGES         15

/************* Ethernet implementation limits ***************************/
#define MAX_ETHERNET_CARDS      10

/********************** Floppy limits ***********************************/
#define MAX_FLOPPY_DRIVES      2

/************* PCI Passthrough implementation limits ********************/
#define MAX_PCI_PASSTHRU_DEVICES 16

/************* Test device implementation limits ********************/
#define MAX_PCI_TEST_DEVICES 16

/************* VDTI PCI Device implementation limits ********************/
#define MAX_VDTI_PCI_DEVICES 16

/************* USB implementation limits ********************************/
#define MAX_USB_DEVICES_PER_HOST_CONTROLLER 127

/************* NVDIMM implementation limits ********************************/
#define NVDIMM_MAX_CONTROLLERS   1
#define MAX_NVDIMM 64

/************* vRDMA implementation limits ******************************/
#define MAX_VRDMA_DEVICES 1

/************* QAT implementation limits ********************/
#define MAX_QAT_PCI_DEVICES 4

/************* Strings for Host USB Driver *******************************/

#ifdef _WIN32

/*
 * Globally unique ID for the VMware device interface. Define INITGUID before including
 * this header file to instantiate the variable.
 */
DEFINE_GUID(GUID_DEVICE_INTERFACE_VMWARE_USB_DEVICES, 
0x2da1fe75, 0xaab3, 0x4d2c, 0xac, 0xdf, 0x39, 0x8, 0x8c, 0xad, 0xa6, 0x65);

/*
 * Globally unique ID for the VMware device setup class.
 */
DEFINE_GUID(GUID_CLASS_VMWARE_USB_DEVICES, 
0x3b3e62a5, 0x3556, 0x4d7e, 0xad, 0xad, 0xf5, 0xfa, 0x3a, 0x71, 0x2b, 0x56);

/*
 * This string defines the device ID string of a VMware USB device.
 * The format is USB\Vid_XXXX&Pid_YYYY, where XXXX and YYYY are the
 * hexadecimal representations of the vendor and product ids, respectively.
 *
 * The official vendor ID for VMware, Inc. is 0x0E0F.
 * The product id for USB generic devices is 0x0001.
 */
#define USB_VMWARE_DEVICE_ID_WIDE L"USB\\Vid_0E0F&Pid_0001"
#define USB_DEVICE_ID_LENGTH (sizeof(USB_VMWARE_DEVICE_ID_WIDE) / sizeof(WCHAR))

#ifdef UNICODE
#define USB_PNP_SETUP_CLASS_NAME L"VMwareUSBDevices"
#define USB_PNP_DRIVER_NAME L"vmusb"
#else
#define USB_PNP_SETUP_CLASS_NAME "VMwareUSBDevices"
#define USB_PNP_DRIVER_NAME "vmusb"
#endif
#endif

/*
 * Our JEDEC 2 Manufacturer ID number is 2 in bank 10.  Our number is nine
 * bytes of continuation code (with an odd parity bit in bit 7) followed by the
 * number itself.
 *
 */
#define JEDEC_VENDOR_ID_VMWARE          0x289
#define JEDEC_DEVICE_ID_VMWARE_NVDIMM   0x1

#endif /* VM_DEVICE_VERSION_H */
�����������������������������������������������������������vmxnet3-only/shared/vmci_defs.h���������������������������������������������������������������������0000444�0000000�0000000�00000066156�13432725350�015535� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2005-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef _VMCI_DEF_H_
#define _VMCI_DEF_H_

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_VMMEXT
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#include "includeCheck.h"

#include "vm_basic_types.h"
#include "vm_basic_defs.h"
#include "vm_atomic.h"
#include "vm_assert.h"

#if defined __cplusplus
extern "C" {
#endif

/* Register offsets. */
#define VMCI_STATUS_ADDR      0x00
#define VMCI_CONTROL_ADDR     0x04
#define VMCI_ICR_ADDR	      0x08
#define VMCI_IMR_ADDR         0x0c
#define VMCI_DATA_OUT_ADDR    0x10
#define VMCI_DATA_IN_ADDR     0x14
#define VMCI_CAPS_ADDR        0x18
#define VMCI_RESULT_LOW_ADDR  0x1c
#define VMCI_RESULT_HIGH_ADDR 0x20

/* Max number of devices. */
#define VMCI_MAX_DEVICES 1

/* Status register bits. */
#define VMCI_STATUS_INT_ON     0x1

/* Control register bits. */
#define VMCI_CONTROL_RESET        0x1
#define VMCI_CONTROL_INT_ENABLE   0x2
#define VMCI_CONTROL_INT_DISABLE  0x4

/* Capabilities register bits. */
#define VMCI_CAPS_HYPERCALL     0x1
#define VMCI_CAPS_GUESTCALL     0x2
#define VMCI_CAPS_DATAGRAM      0x4
#define VMCI_CAPS_NOTIFICATIONS 0x8

/* Interrupt Cause register bits. */
#define VMCI_ICR_DATAGRAM      0x1
#define VMCI_ICR_NOTIFICATION  0x2

/* Interrupt Mask register bits. */
#define VMCI_IMR_DATAGRAM      0x1
#define VMCI_IMR_NOTIFICATION  0x2

/* Interrupt type. */
typedef enum VMCIIntrType {
   VMCI_INTR_TYPE_INTX = 0,
   VMCI_INTR_TYPE_MSI =  1,
   VMCI_INTR_TYPE_MSIX = 2
} VMCIIntrType;

/*
 * Maximum MSI/MSI-X interrupt vectors in the device.
 */
#define VMCI_MAX_INTRS 2

/*
 * Supported interrupt vectors.  There is one for each ICR value above,
 * but here they indicate the position in the vector array/message ID.
 */
#define VMCI_INTR_DATAGRAM     0
#define VMCI_INTR_NOTIFICATION 1

/*
 * A single VMCI device has an upper limit of 128 MiB on the amount of
 * memory that can be used for queue pairs.
 */
#define VMCI_MAX_GUEST_QP_MEMORY (128 * 1024 * 1024)

/*
 * We have a fixed set of resource IDs available in the VMX.
 * This allows us to have a very simple implementation since we statically
 * know how many will create datagram handles. If a new caller arrives and
 * we have run out of slots we can manually increment the maximum size of
 * available resource IDs.
 */

typedef uint32 VMCI_Resource;

/* VMCI reserved hypervisor datagram resource IDs. */
#define VMCI_RESOURCES_QUERY        0
#define VMCI_GET_CONTEXT_ID         1
#define VMCI_SET_NOTIFY_BITMAP      2
#define VMCI_DOORBELL_LINK          3
#define VMCI_DOORBELL_UNLINK        4
#define VMCI_DOORBELL_NOTIFY        5
/*
 * VMCI_DATAGRAM_REQUEST_MAP and VMCI_DATAGRAM_REMOVE_MAP are
 * obsoleted by the removal of VM to VM communication.
 */
#define VMCI_DATAGRAM_REQUEST_MAP   6
#define VMCI_DATAGRAM_REMOVE_MAP    7
#define VMCI_EVENT_SUBSCRIBE        8
#define VMCI_EVENT_UNSUBSCRIBE      9
#define VMCI_QUEUEPAIR_ALLOC        10
#define VMCI_QUEUEPAIR_DETACH       11
/*
 * VMCI_VSOCK_VMX_LOOKUP was assigned to 12 for Fusion 3.0/3.1,
 * WS 7.0/7.1 and ESX 4.1
 */
#define VMCI_HGFS_TRANSPORT         13
#define VMCI_UNITY_PBRPC_REGISTER   14
/*
 * This resource is used for VMCI socket control packets sent to the
 * hypervisor (CID 0) because RID 1 is already reserved.
 */
#define VSOCK_PACKET_HYPERVISOR_RID 15
#define VMCI_RESOURCE_MAX           16
/*
 * The core VMCI device functionality only requires the resource IDs of
 * VMCI_QUEUEPAIR_DETACH and below.
 */
#define VMCI_CORE_DEVICE_RESOURCE_MAX  VMCI_QUEUEPAIR_DETACH

/*
 * VMCI reserved host datagram resource IDs.
 * vsock control channel has resource id 1.
 */
#define VMCI_DVFILTER_DATA_PATH_DATAGRAM 2

/* VMCI Ids. */
typedef uint32 VMCIId;

typedef struct VMCIIdRange {
   int8 action;   // VMCI_FA_X, for use in filters.
   VMCIId begin;  // Beginning of range
   VMCIId end;    // End of range
} VMCIIdRange;

typedef struct VMCIHandle {
   VMCIId context;
   VMCIId resource;
} VMCIHandle;

static INLINE VMCIHandle
VMCI_MAKE_HANDLE(VMCIId cid,  // IN:
                 VMCIId rid)  // IN:
{
   VMCIHandle h;
   h.context = cid;
   h.resource = rid;
   return h;
}

/*
 *----------------------------------------------------------------------
 *
 * VMCI_HANDLE_TO_UINT64 --
 *
 *     Helper for VMCI handle to uint64 conversion.
 *
 * Results:
 *     The uint64 value.
 *
 * Side effects:
 *     None.
 *
 *----------------------------------------------------------------------
 */

static INLINE uint64
VMCI_HANDLE_TO_UINT64(VMCIHandle handle) // IN:
{
   uint64 handle64;

handle64 = handle.context;
   handle64 <<= 32;
   handle64 |= handle.resource;
   return handle64;
}

/*
 *----------------------------------------------------------------------
 *
 * VMCI_UINT64_TO_HANDLE --
 *
 *     Helper for uint64 to VMCI handle conversion.
 *
 * Results:
 *     The VMCI handle value.
 *
 * Side effects:
 *     None.
 *
 *----------------------------------------------------------------------
 */

static INLINE VMCIHandle
VMCI_UINT64_TO_HANDLE(uint64 handle64) // IN:
{
   VMCIId context = (VMCIId)(handle64 >> 32);
   VMCIId resource = (VMCIId)handle64;

return VMCI_MAKE_HANDLE(context, resource);
}

#define VMCI_HANDLE_TO_CONTEXT_ID(_handle) ((_handle).context)
#define VMCI_HANDLE_TO_RESOURCE_ID(_handle) ((_handle).resource)
#define VMCI_HANDLE_EQUAL(_h1, _h2) ((_h1).context == (_h2).context && \
				     (_h1).resource == (_h2).resource)

#define VMCI_INVALID_ID 0xFFFFFFFF
static const VMCIHandle VMCI_INVALID_HANDLE = {VMCI_INVALID_ID,
					       VMCI_INVALID_ID};

#define VMCI_HANDLE_INVALID(_handle)   \
   VMCI_HANDLE_EQUAL((_handle), VMCI_INVALID_HANDLE)

/*
 * The below defines can be used to send anonymous requests.
 * This also indicates that no response is expected.
 */
#define VMCI_ANON_SRC_CONTEXT_ID   VMCI_INVALID_ID
#define VMCI_ANON_SRC_RESOURCE_ID  VMCI_INVALID_ID
#define VMCI_ANON_SRC_HANDLE       VMCI_MAKE_HANDLE(VMCI_ANON_SRC_CONTEXT_ID, \
						    VMCI_ANON_SRC_RESOURCE_ID)

/* The lowest 16 context ids are reserved for internal use. */
#define VMCI_RESERVED_CID_LIMIT 16

/*
 * Hypervisor context id, used for calling into hypervisor
 * supplied services from the VM.
 */
#define VMCI_HYPERVISOR_CONTEXT_ID 0

/*
 * Well-known context id, a logical context that contains a set of
 * well-known services. This context ID is now obsolete.
 */
#define VMCI_WELL_KNOWN_CONTEXT_ID 1

/*
 * Context ID used by host endpoints.
 */
#define VMCI_HOST_CONTEXT_ID  2
#define VMCI_HOST_CONTEXT_INVALID_EVENT         ((uintptr_t)~0)

#define VMCI_CONTEXT_IS_VM(_cid) (VMCI_INVALID_ID != _cid && \
                                  _cid > VMCI_HOST_CONTEXT_ID)

/*
 * The VMCI_CONTEXT_RESOURCE_ID is used together with VMCI_MAKE_HANDLE to make
 * handles that refer to a specific context.
 */
#define VMCI_CONTEXT_RESOURCE_ID 0

/*
 *-----------------------------------------------------------------------------
 *
 * VMCI error codes.
 *
 *-----------------------------------------------------------------------------
 */

#define VMCI_SUCCESS_QUEUEPAIR_ATTACH     5
#define VMCI_SUCCESS_QUEUEPAIR_CREATE     4
#define VMCI_SUCCESS_LAST_DETACH          3
#define VMCI_SUCCESS_ACCESS_GRANTED       2
#define VMCI_SUCCESS_ENTRY_DEAD           1
#define VMCI_SUCCESS                      0LL
#define VMCI_ERROR_INVALID_RESOURCE      (-1)
#define VMCI_ERROR_INVALID_ARGS          (-2)
#define VMCI_ERROR_NO_MEM                (-3)
#define VMCI_ERROR_DATAGRAM_FAILED       (-4)
#define VMCI_ERROR_MORE_DATA             (-5)
#define VMCI_ERROR_NO_MORE_DATAGRAMS     (-6)
#define VMCI_ERROR_NO_ACCESS             (-7)
#define VMCI_ERROR_NO_HANDLE             (-8)
#define VMCI_ERROR_DUPLICATE_ENTRY       (-9)
#define VMCI_ERROR_DST_UNREACHABLE       (-10)
#define VMCI_ERROR_PAYLOAD_TOO_LARGE     (-11)
#define VMCI_ERROR_INVALID_PRIV          (-12)
#define VMCI_ERROR_GENERIC               (-13)
#define VMCI_ERROR_PAGE_ALREADY_SHARED   (-14)
#define VMCI_ERROR_CANNOT_SHARE_PAGE     (-15)
#define VMCI_ERROR_CANNOT_UNSHARE_PAGE   (-16)
#define VMCI_ERROR_NO_PROCESS            (-17)
#define VMCI_ERROR_NO_DATAGRAM           (-18)
#define VMCI_ERROR_NO_RESOURCES          (-19)
#define VMCI_ERROR_UNAVAILABLE           (-20)
#define VMCI_ERROR_NOT_FOUND             (-21)
#define VMCI_ERROR_ALREADY_EXISTS        (-22)
#define VMCI_ERROR_NOT_PAGE_ALIGNED      (-23)
#define VMCI_ERROR_INVALID_SIZE          (-24)
#define VMCI_ERROR_REGION_ALREADY_SHARED (-25)
#define VMCI_ERROR_TIMEOUT               (-26)
#define VMCI_ERROR_DATAGRAM_INCOMPLETE   (-27)
#define VMCI_ERROR_INCORRECT_IRQL        (-28)
#define VMCI_ERROR_EVENT_UNKNOWN         (-29)
#define VMCI_ERROR_OBSOLETE              (-30)
#define VMCI_ERROR_QUEUEPAIR_MISMATCH    (-31)
#define VMCI_ERROR_QUEUEPAIR_NOTSET      (-32)
#define VMCI_ERROR_QUEUEPAIR_NOTOWNER    (-33)
#define VMCI_ERROR_QUEUEPAIR_NOTATTACHED (-34)
#define VMCI_ERROR_QUEUEPAIR_NOSPACE     (-35)
#define VMCI_ERROR_QUEUEPAIR_NODATA      (-36)
#define VMCI_ERROR_BUSMEM_INVALIDATION   (-37)
#define VMCI_ERROR_MODULE_NOT_LOADED     (-38)
#define VMCI_ERROR_DEVICE_NOT_FOUND      (-39)
#define VMCI_ERROR_QUEUEPAIR_NOT_READY   (-40)
#define VMCI_ERROR_WOULD_BLOCK           (-41)

/* VMCI clients should return error code withing this range */
#define VMCI_ERROR_CLIENT_MIN     (-500)
#define VMCI_ERROR_CLIENT_MAX     (-550)

/* Internal error codes. */
#define VMCI_SHAREDMEM_ERROR_BAD_CONTEXT (-1000)

#define VMCI_PATH_MAX 256

/* VMCI reserved events. */
typedef uint32 VMCI_Event;

#define VMCI_EVENT_CTX_ID_UPDATE  0  // Only applicable to guest endpoints
#define VMCI_EVENT_CTX_REMOVED    1  // Applicable to guest and host
#define VMCI_EVENT_QP_RESUMED     2  // Only applicable to guest endpoints
#define VMCI_EVENT_QP_PEER_ATTACH 3  // Applicable to guest, host and VMX
#define VMCI_EVENT_QP_PEER_DETACH 4  // Applicable to guest, host and VMX
#define VMCI_EVENT_MEM_ACCESS_ON  5  // Applicable to VMX and vmk.  On vmk,
                                     // this event has the Context payload type.
#define VMCI_EVENT_MEM_ACCESS_OFF 6  // Applicable to VMX and vmk.  Same as
                                     // above for the payload type.
#define VMCI_EVENT_GUEST_PAUSED   7  // Applicable to vmk. This event has the
                                     // Context payload type.
#define VMCI_EVENT_GUEST_UNPAUSED 8  // Applicable to vmk. Same as above for
                                     // the payload type.
#define VMCI_EVENT_MAX            9

/*
 * Of the above events, a few are reserved for use in the VMX, and
 * other endpoints (guest and host kernel) should not use them. For
 * the rest of the events, we allow both host and guest endpoints to
 * subscribe to them, to maintain the same API for host and guest
 * endpoints.
 */

#define VMCI_EVENT_VALID_VMX(_event) (_event == VMCI_EVENT_QP_PEER_ATTACH || \
                                      _event == VMCI_EVENT_QP_PEER_DETACH || \
                                      _event == VMCI_EVENT_MEM_ACCESS_ON || \
                                      _event == VMCI_EVENT_MEM_ACCESS_OFF)

#if defined(VMX86_SERVER)
#define VMCI_EVENT_VALID(_event) (_event < VMCI_EVENT_MAX)
#else // VMX86_SERVER
#define VMCI_EVENT_VALID(_event) (_event < VMCI_EVENT_MAX && \
                                  _event != VMCI_EVENT_MEM_ACCESS_ON && \
                                  _event != VMCI_EVENT_MEM_ACCESS_OFF && \
                                  _event != VMCI_EVENT_GUEST_PAUSED && \
                                  _event != VMCI_EVENT_GUEST_UNPAUSED)
#endif // VMX86_SERVER

/* Reserved guest datagram resource ids. */
#define VMCI_EVENT_HANDLER 0

/* VMCI privileges. */
typedef enum VMCIResourcePrivilegeType {
   VMCI_PRIV_CH_PRIV,
   VMCI_PRIV_DESTROY_RESOURCE,
   VMCI_PRIV_ASSIGN_CLIENT,
   VMCI_PRIV_DG_CREATE,
   VMCI_PRIV_DG_SEND,
   VMCI_PRIV_NOTIFY,
   VMCI_NUM_PRIVILEGES,
} VMCIResourcePrivilegeType;

/*
 * VMCI coarse-grained privileges (per context or host
 * process/endpoint. An entity with the restricted flag is only
 * allowed to interact with the hypervisor and trusted entities.
 */
typedef uint32 VMCIPrivilegeFlags;

#define VMCI_PRIVILEGE_FLAG_RESTRICTED     0x01
#define VMCI_PRIVILEGE_FLAG_TRUSTED        0x02
#define VMCI_PRIVILEGE_ALL_FLAGS           (VMCI_PRIVILEGE_FLAG_RESTRICTED | \
                                            VMCI_PRIVILEGE_FLAG_TRUSTED)
#define VMCI_NO_PRIVILEGE_FLAGS            0x00
#define VMCI_DEFAULT_PROC_PRIVILEGE_FLAGS  VMCI_NO_PRIVILEGE_FLAGS
#define VMCI_LEAST_PRIVILEGE_FLAGS         VMCI_PRIVILEGE_FLAG_RESTRICTED
#define VMCI_MAX_PRIVILEGE_FLAGS           VMCI_PRIVILEGE_FLAG_TRUSTED

#define VMCI_PUBLIC_GROUP_NAME "vmci public group"
/* 0 through VMCI_RESERVED_RESOURCE_ID_MAX are reserved. */
#define VMCI_RESERVED_RESOURCE_ID_MAX 1023

#define VMCI_DOMAIN_NAME_MAXLEN  32

#define VMCI_LGPFX "VMCI: "

/*
 * VMCIQueueHeader
 *
 * A Queue cannot stand by itself as designed.  Each Queue's header
 * contains a pointer into itself (the producerTail) and into its peer
 * (consumerHead).  The reason for the separation is one of
 * accessibility: Each end-point can modify two things: where the next
 * location to enqueue is within its produceQ (producerTail); and
 * where the next dequeue location is in its consumeQ (consumerHead).
 *
 * An end-point cannot modify the pointers of its peer (guest to
 * guest; NOTE that in the host both queue headers are mapped r/w).
 * But, each end-point needs read access to both Queue header
 * structures in order to determine how much space is used (or left)
 * in the Queue.  This is because for an end-point to know how full
 * its produceQ is, it needs to use the consumerHead that points into
 * the produceQ but -that- consumerHead is in the Queue header for
 * that end-points consumeQ.
 *
 * Thoroughly confused?  Sorry.
 *
 * producerTail: the point to enqueue new entrants.  When you approach
 * a line in a store, for example, you walk up to the tail.
 *
 * consumerHead: the point in the queue from which the next element is
 * dequeued.  In other words, who is next in line is he who is at the
 * head of the line.
 *
 * Also, producerTail points to an empty byte in the Queue, whereas
 * consumerHead points to a valid byte of data (unless producerTail ==
 * consumerHead in which case consumerHead does not point to a valid
 * byte of data).
 *
 * For a queue of buffer 'size' bytes, the tail and head pointers will be in
 * the range [0, size-1].
 *
 * If produceQHeader->producerTail == consumeQHeader->consumerHead
 * then the produceQ is empty.
 */

typedef struct VMCIQueueHeader {
   /* All fields are 64bit and aligned. */
   VMCIHandle    handle;       /* Identifier. */
   Atomic_uint64 producerTail; /* Offset in this queue. */
   Atomic_uint64 consumerHead; /* Offset in peer queue. */
} VMCIQueueHeader;

/*
 * If one client of a QueuePair is a 32bit entity, we restrict the QueuePair
 * size to be less than 4GB, and use 32bit atomic operations on the head and
 * tail pointers. 64bit atomic read on a 32bit entity involves cmpxchg8b which
 * is an atomic read-modify-write. This will cause traces to fire when a 32bit
 * consumer tries to read the producer's tail pointer, for example, because the
 * consumer has read-only access to the producer's tail pointer.
 *
 * We provide the following macros to invoke 32bit or 64bit atomic operations
 * based on the architecture the code is being compiled on.
 */

/* Architecture independent maximum queue size. */
#define QP_MAX_QUEUE_SIZE_ARCH_ANY   CONST64U(0xffffffff)

#ifdef __x86_64__
#  define QP_MAX_QUEUE_SIZE_ARCH     CONST64U(0xffffffffffffffff)
#  define QPAtomic_ReadOffset(x)     Atomic_Read64(x)
#  define QPAtomic_WriteOffset(x, y) Atomic_Write64(x, y)
#else
   /*
    * Wrappers below are being used to call Atomic_Read32 because of the
    * 'type punned' compilation warning received when Atomic_Read32 is
    * called with a Atomic_uint64 pointer typecasted to Atomic_uint32
    * pointer from QPAtomic_ReadOffset. Ditto with QPAtomic_WriteOffset.
    */

static INLINE uint32
   TypeSafe_Atomic_Read32(void *var) // IN:
   {
      return Atomic_Read32((Atomic_uint32 *)(var));
   }

static INLINE void
   TypeSafe_Atomic_Write32(void *var, uint32 val) // IN:
   {
      Atomic_Write32((Atomic_uint32 *)(var), (uint32)(val));
   }

#  define QP_MAX_QUEUE_SIZE_ARCH  CONST64U(0xffffffff)
#  define QPAtomic_ReadOffset(x)  TypeSafe_Atomic_Read32((void *)(x))
#  define QPAtomic_WriteOffset(x, y) \
          TypeSafe_Atomic_Write32((void *)(x), (uint32)(y))
#endif	/* __x86_64__  */

/*
 *-----------------------------------------------------------------------------
 *
 * QPAddPointer --
 *
 *      Helper to add a given offset to a head or tail pointer. Wraps the value
 *      of the pointer around the max size of the queue.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
QPAddPointer(Atomic_uint64 *var, // IN:
             size_t add,         // IN:
             uint64 size)        // IN:
{
   uint64 newVal = QPAtomic_ReadOffset(var);

if (newVal >= size - add) {
      newVal -= size;
   }
   newVal += add;

QPAtomic_WriteOffset(var, newVal);
}

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIQueueHeader_ProducerTail() --
 *
 *      Helper routine to get the Producer Tail from the supplied queue.
 *
 * Results:
 *      The contents of the queue's producer tail.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint64
VMCIQueueHeader_ProducerTail(const VMCIQueueHeader *qHeader) // IN:
{
   VMCIQueueHeader *qh = (VMCIQueueHeader *)qHeader;
   return QPAtomic_ReadOffset(&qh->producerTail);
}

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIQueueHeader_ConsumerHead() --
 *
 *      Helper routine to get the Consumer Head from the supplied queue.
 *
 * Results:
 *      The contents of the queue's consumer tail.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint64
VMCIQueueHeader_ConsumerHead(const VMCIQueueHeader *qHeader) // IN:
{
   VMCIQueueHeader *qh = (VMCIQueueHeader *)qHeader;
   return QPAtomic_ReadOffset(&qh->consumerHead);
}

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIQueueHeader_AddProducerTail() --
 *
 *      Helper routine to increment the Producer Tail.  Fundamentally,
 *      QPAddPointer() is used to manipulate the tail itself.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
VMCIQueueHeader_AddProducerTail(VMCIQueueHeader *qHeader, // IN/OUT:
                                size_t add,               // IN:
                                uint64 queueSize)         // IN:
{
   QPAddPointer(&qHeader->producerTail, add, queueSize);
}

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIQueueHeader_AddConsumerHead() --
 *
 *      Helper routine to increment the Consumer Head.  Fundamentally,
 *      QPAddPointer() is used to manipulate the head itself.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
VMCIQueueHeader_AddConsumerHead(VMCIQueueHeader *qHeader, // IN/OUT:
                                size_t add,               // IN:
                                uint64 queueSize)         // IN:
{
   QPAddPointer(&qHeader->consumerHead, add, queueSize);
}

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIQueueHeader_CheckAlignment --
 *
 *      Checks if the given queue is aligned to page boundary.  Returns TRUE if
 *      the alignment is good.
 *
 * Results:
 *      TRUE or FALSE.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE Bool
VMCIQueueHeader_CheckAlignment(const VMCIQueueHeader *qHeader) // IN:
{
   uintptr_t hdr, offset;

hdr = (uintptr_t) qHeader;
   offset = hdr & (PAGE_SIZE -1);

return offset == 0;
}

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIQueueHeader_GetPointers --
 *
 *      Helper routine for getting the head and the tail pointer for a queue.
 *      Both the VMCIQueues are needed to get both the pointers for one queue.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
VMCIQueueHeader_GetPointers(const VMCIQueueHeader *produceQHeader, // IN:
                            const VMCIQueueHeader *consumeQHeader, // IN:
                            uint64 *producerTail,                  // OUT:
                            uint64 *consumerHead)                  // OUT:
{
   if (producerTail) {
      *producerTail = VMCIQueueHeader_ProducerTail(produceQHeader);
   }

if (consumerHead) {
      *consumerHead = VMCIQueueHeader_ConsumerHead(consumeQHeader);
   }
}

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIQueueHeader_ResetPointers --
 *
 *      Reset the tail pointer (of "this" queue) and the head pointer (of
 *      "peer" queue).
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
VMCIQueueHeader_ResetPointers(VMCIQueueHeader *qHeader) // IN/OUT:
{
   QPAtomic_WriteOffset(&qHeader->producerTail, CONST64U(0));
   QPAtomic_WriteOffset(&qHeader->consumerHead, CONST64U(0));
}

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIQueueHeader_Init --
 *
 *      Initializes a queue's state (head & tail pointers).
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
VMCIQueueHeader_Init(VMCIQueueHeader *qHeader, // IN/OUT:
                     const VMCIHandle handle)  // IN:
{
   qHeader->handle = handle;
   VMCIQueueHeader_ResetPointers(qHeader);
}

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIQueueHeader_FreeSpace --
 *
 *      Finds available free space in a produce queue to enqueue more
 *      data or reports an error if queue pair corruption is detected.
 *
 * Results:
 *      Free space size in bytes or an error code.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE int64
VMCIQueueHeader_FreeSpace(const VMCIQueueHeader *produceQHeader, // IN:
                          const VMCIQueueHeader *consumeQHeader, // IN:
                          const uint64 produceQSize)             // IN:
{
   uint64 tail;
   uint64 head;
   uint64 freeSpace;

tail = VMCIQueueHeader_ProducerTail(produceQHeader);
   head = VMCIQueueHeader_ConsumerHead(consumeQHeader);

if (tail >= produceQSize || head >= produceQSize) {
      return VMCI_ERROR_INVALID_SIZE;
   }

/*
    * Deduct 1 to avoid tail becoming equal to head which causes ambiguity. If
    * head and tail are equal it means that the queue is empty.
    */

if (tail >= head) {
      freeSpace = produceQSize - (tail - head) - 1;
   } else {
      freeSpace = head - tail - 1;
   }

return freeSpace;
}

/*
 *-----------------------------------------------------------------------------
 *
 * VMCIQueueHeader_BufReady --
 *
 *      VMCIQueueHeader_FreeSpace() does all the heavy lifting of
 *      determing the number of free bytes in a Queue.  This routine,
 *      then subtracts that size from the full size of the Queue so
 *      the caller knows how many bytes are ready to be dequeued.
 *
 * Results:
 *      On success, available data size in bytes (up to MAX_INT64).
 *      On failure, appropriate error code.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE int64
VMCIQueueHeader_BufReady(const VMCIQueueHeader *consumeQHeader, // IN:
                         const VMCIQueueHeader *produceQHeader, // IN:
                         const uint64 consumeQSize)             // IN:
{
   int64 freeSpace;

freeSpace = VMCIQueueHeader_FreeSpace(consumeQHeader,
                                         produceQHeader,
                                         consumeQSize);
   if (freeSpace < VMCI_SUCCESS) {
      return freeSpace;
   } else {
      return consumeQSize - freeSpace - 1;
   }
}

/*
 * Defines for the VMCI traffic filter:
 * - VMCI_FA_<name> defines the filter action values
 * - VMCI_FP_<name> defines the filter protocol values
 * - VMCI_FD_<name> defines the direction values (guest or host)
 * - VMCI_FT_<name> are the type values (allow or deny)
 */

#define VMCI_FA_INVALID -1
#define VMCI_FA_ALLOW    0
#define VMCI_FA_DENY     (VMCI_FA_ALLOW + 1)
#define VMCI_FA_MAX      (VMCI_FA_DENY + 1)

#define VMCI_FP_INVALID     -1
#define VMCI_FP_HYPERVISOR   0
#define VMCI_FP_QUEUEPAIR    (VMCI_FP_HYPERVISOR + 1)
#define VMCI_FP_DOORBELL     (VMCI_FP_QUEUEPAIR + 1)
#define VMCI_FP_DATAGRAM     (VMCI_FP_DOORBELL + 1)
#define VMCI_FP_STREAMSOCK   (VMCI_FP_DATAGRAM + 1)
#define VMCI_FP_ANY          (VMCI_FP_STREAMSOCK + 1)
#define VMCI_FP_MAX          (VMCI_FP_ANY + 1)

#define VMCI_FD_INVALID  -1
#define VMCI_FD_GUEST     0
#define VMCI_FD_HOST      (VMCI_FD_GUEST + 1)
#define VMCI_FD_ANY       (VMCI_FD_HOST + 1)
#define VMCI_FD_MAX       (VMCI_FD_ANY + 1)

/*
 * The filter list tracks VMCI Id ranges for a given filter.
 */

typedef struct {
   uint32 len;
   VMCIIdRange *list;
} VMCIFilterList;

/*
 * The filter info is used to communicate the filter configuration
 * from the VMX to the host kernel.
 */

typedef struct {
   VA64   list;   // List of VMCIIdRange
   uint32 len;    // Length of list
   uint8  dir;    // VMCI_FD_X
   uint8  proto;  // VMCI_FP_X
} VMCIFilterInfo;

/*
 * In the host kernel, the ingoing and outgoing filters are
 * separated. The VMCIProtoFilters type captures all filters in one
 * direction. The VMCIFilters type captures all filters.
 */

typedef VMCIFilterList VMCIProtoFilters[VMCI_FP_MAX];
typedef VMCIProtoFilters VMCIFilters[VMCI_FD_MAX];

#if defined __cplusplus
} // extern "C"
#endif

#endif // _VMCI_DEF_H_
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_version.h����������������������������������������������������������������0000444�0000000�0000000�00000007363�13432725347�016627� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_VERSION_H__
#   define __COMPAT_VERSION_H__

#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMKDRIVERS
#include "includeCheck.h"

#ifndef __linux__
#   error "linux-version.h"
#endif

#include <linux/version.h>

#ifndef KERNEL_VERSION
#   error KERNEL_VERSION macro is not defined, environment is busted
#endif

/*
 * Distinguish relevant classes of Linux kernels.
 *
 * The convention is that version X defines all
 * the KERNEL_Y symbols where Y <= X.
 *
 * XXX Do not add more definitions here. This way of doing things does not
 *     scale, and we are going to phase it out soon --hpreg
 */

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 1, 0)
#   define KERNEL_2_1
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 2, 0)
#   define KERNEL_2_2
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 3, 1)
#   define KERNEL_2_3_1
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 3, 15)
/*   new networking */
#   define KERNEL_2_3_15
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 3, 25)
/*  new procfs */
#   define KERNEL_2_3_25
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 3, 29)
/*  even newer procfs */
#   define KERNEL_2_3_29
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 3, 43)
/*  softnet changes */
#   define KERNEL_2_3_43
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 3, 47)
/*  more softnet changes */
#   define KERNEL_2_3_47
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 3, 99)
/*  name in netdevice struct is array and not pointer */
#   define KERNEL_2_3_99
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 0)
/*  New 'owner' member at the beginning of struct file_operations */
#      define KERNEL_2_4_0
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 8)
/*  New netif_rx_ni() --hpreg */
#   define KERNEL_2_4_8
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 2)
/*  New kdev_t, major()/minor() API --hpreg */
#   define KERNEL_2_5_2
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 5)
/*  New sk_alloc(), pte_offset_map()/pte_unmap() --hpreg */
#   define KERNEL_2_5_5
#endif

/* Linux kernel 3.0 can be called 2.6.40, and 3.1 can be 2.6.41...
 * Use COMPAT_LINUX_VERSION_CHECK_LT iff you need to compare running kernel to
 * versions 3.0 and above.
 *
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 0, 0)
   /* Straight forward comparison if kernel version is 3.0.0 and beyond */
#   define COMPAT_LINUX_VERSION_CHECK_LT(a, b, c) LINUX_VERSION_CODE < KERNEL_VERSION (a, b, c)
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 40)
   /* Use b of the check to calculate corresponding c of kernel
    *  version to compare */
#   define COMPAT_LINUX_VERSION_CHECK_LT(a, b, c) LINUX_VERSION_CODE < KERNEL_VERSION (2, 6, (b + 40))
#else
    /* This is anyways lesser than any 3.x versions */
#   define COMPAT_LINUX_VERSION_CHECK_LT(a, b, c) 1
#endif

#endif /* __COMPAT_VERSION_H__ */
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_pgtable.h����������������������������������������������������������������0000444�0000000�0000000�00000005563�13432725347�016560� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_PGTABLE_H__
#   define __COMPAT_PGTABLE_H__

#if defined(CONFIG_PARAVIRT) && defined(CONFIG_HIGHPTE)
#   if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 21)
#      include <asm/paravirt.h>
#      undef paravirt_map_pt_hook
#      define paravirt_map_pt_hook(type, va, pfn) do {} while (0)
#   endif
#endif
#include <asm/pgtable.h>

/*
 * p4d level appeared in 4.12.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0)
#   define compat_p4d_offset(pgd, address)  p4d_offset(pgd, address)
#   define compat_p4d_present(p4d)          p4d_present(p4d)
#   define compat_p4d_large(p4d)            p4d_large(p4d)
#   define compat_p4d_pfn(p4d)              p4d_pfn(p4d)
#   define COMPAT_P4D_MASK                  P4D_MASK
typedef p4d_t compat_p4d_t;
#else
#   define compat_p4d_offset(pgd, address)  (pgd)
#   define compat_p4d_present(p4d)          (1)
#   define compat_p4d_large(p4d)            (0)
#   define compat_p4d_pfn(p4d)              INVALID_MPN  /* Not used */
#   define COMPAT_P4D_MASK                  0            /* Not used */
typedef pgd_t compat_p4d_t;
#endif
/* pud_pfn did not exist before 3.8. */
#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0)
#   define pud_pfn(pud)  INVALID_MPN
#endif

/*
 * Define VM_PAGE_KERNEL_EXEC for vmapping executable pages.
 *
 * On ia32 PAGE_KERNEL_EXEC was introduced in 2.6.8.1.  Unfortunately it accesses
 * __PAGE_KERNEL_EXEC which is not exported for modules.  So we use
 * __PAGE_KERNEL and just cut _PAGE_NX bit from it.
 *
 * For ia32 kernels before 2.6.8.1 we use PAGE_KERNEL directly, these kernels
 * do not have noexec support.
 *
 * On x86-64 situation is a bit better: they always supported noexec, but
 * before 2.6.8.1 flag was named PAGE_KERNEL_EXECUTABLE, and it was renamed
 * to PAGE_KERNEL_EXEC when ia32 got noexec too (see above).
 */
#ifdef CONFIG_X86
#ifdef _PAGE_NX
#define VM_PAGE_KERNEL_EXEC __pgprot(__PAGE_KERNEL & ~_PAGE_NX)
#else
#define VM_PAGE_KERNEL_EXEC PAGE_KERNEL
#endif
#else
#define VM_PAGE_KERNEL_EXEC PAGE_KERNEL_EXEC
#endif

#endif /* __COMPAT_PGTABLE_H__ */
���������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_page.h�������������������������������������������������������������������0000444�0000000�0000000�00000004663�13432725347�016056� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_PAGE_H__
#   define __COMPAT_PAGE_H__

#include <linux/mm.h>
#include <asm/page.h>

/* The pfn_to_page() API appeared in 2.5.14 and changed to function during 2.6.x */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0) && !defined(pfn_to_page)
#   define pfn_to_page(_pfn) (mem_map + (_pfn))
#   define page_to_pfn(_page) ((_page) - mem_map)
#endif

/* The virt_to_page() API appeared in 2.4.0 --hpreg */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 0) && !defined(virt_to_page)
#   define virt_to_page(_kvAddr) pfn_to_page(MAP_NR(_kvAddr))
#endif

/*
 * The get_order() API appeared at some point in 2.3.x, and was then backported
 * in 2.2.17-21mdk and in the stock 2.2.18. Because we can only detect its
 * definition through makefile tricks, we provide our own for now --hpreg
 */
static inline int
compat_get_order(unsigned long size) // IN
{
   int order;

size = (size - 1) >> (PAGE_SHIFT - 1);
   order = -1;
   do {
      size >>= 1;
      order++;
   } while (size);

return order;
}

/* 
 * BUG() was added to <asm/page.h> in 2.2.18, and was moved to <asm/bug.h>
 * in 2.5.58.
 * 
 * XXX: Technically, this belongs in some sort of "compat_asm_page.h" file, but
 * since our compatibility wrappers don't distinguish between <asm/xxx.h> and
 * <linux/xxx.h>, putting it here is reasonable.
 */
#ifndef BUG
#define BUG() do {                                                            \
   printk("kernel BUG at %s:%d!\n", __FILE__, __LINE__);                      \
  __asm__ __volatile__(".byte 0x0f,0x0b");                                    \
} while (0)
#endif

#endif /* __COMPAT_PAGE_H__ */
�����������������������������������������������������������������������������vmxnet3-only/shared/includeCheck.h������������������������������������������������������������������0000444�0000000�0000000�00000007463�13432725350�016153� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * includeCheck.h --
 *
 *	Restrict include file use.
 *
 * In every .h file, define one or more of these
 *
 *	INCLUDE_ALLOW_VMX 
 *	INCLUDE_ALLOW_USERLEVEL 
 *	INCLUDE_ALLOW_VMCORE
 *	INCLUDE_ALLOW_MODULE
 *	INCLUDE_ALLOW_VMKERNEL 
 *	INCLUDE_ALLOW_DISTRIBUTE
 *	INCLUDE_ALLOW_VMK_MODULE
 *      INCLUDE_ALLOW_VMKDRIVERS
 *      INCLUDE_ALLOW_MKS
 *
 * Then include this file.
 *
 * Any file that has INCLUDE_ALLOW_DISTRIBUTE defined will potentially
 * be distributed in source form along with GPLed code.  Ensure
 * that this is acceptable.
 */

/*
 * Declare a VMCORE-only variable to help classify object
 * files.  The variable goes in the common block and does
 * not create multiple definition link-time conflicts.
 */

#if defined VMCORE && defined VMX86_DEVEL && defined VMX86_DEBUG && \
    defined linux && !defined MODULE && \
    !defined COMPILED_WITH_VMCORE
#define COMPILED_WITH_VMCORE compiled_with_vmcore
#ifdef ASM
        .comm   compiled_with_vmcore, 0
#else
        asm(".comm compiled_with_vmcore, 0");
#endif /* ASM */
#endif

#if defined VMCORE && \
    !(defined VMX86_VMX || defined VMM || \
      defined MONITOR_APP || defined VMMON)
#error "Makefile problem: VMCORE without VMX86_VMX or \
        VMM or MONITOR_APP or MODULE."
#endif

#if defined VMCORE && !defined INCLUDE_ALLOW_VMCORE
#error "The surrounding include file is not allowed in vmcore."
#endif
#undef INCLUDE_ALLOW_VMCORE

#if defined VMX86_VMX && !defined VMCORE && \
    !defined INCLUDE_ALLOW_VMX && !defined INCLUDE_ALLOW_USERLEVEL && \
    !defined INCLUDE_ALLOW_MKS
#error "The surrounding include file is not allowed in the VMX."
#endif
#undef INCLUDE_ALLOW_VMX

#if defined USERLEVEL && !defined VMX86_VMX && !defined VMCORE && \
    !defined INCLUDE_ALLOW_USERLEVEL && !defined INCLUDE_ALLOW_MKS
#error "The surrounding include file is not allowed at userlevel."
#endif
#undef INCLUDE_ALLOW_USERLEVEL

#if defined MODULE && !defined VMKERNEL_MODULE && \
    !defined VMMON && !defined INCLUDE_ALLOW_MODULE
#error "The surrounding include file is not allowed in driver modules."
#endif
#undef INCLUDE_ALLOW_MODULE

#if defined VMMON && !defined INCLUDE_ALLOW_VMMON
#error "The surrounding include file is not allowed in vmmon."
#endif
#undef INCLUDE_ALLOW_VMMON

#if defined VMKERNEL && !defined INCLUDE_ALLOW_VMKERNEL
#error "The surrounding include file is not allowed in the vmkernel."
#endif
#undef INCLUDE_ALLOW_VMKERNEL

#if defined GPLED_CODE && !defined INCLUDE_ALLOW_DISTRIBUTE
#error "The surrounding include file is not allowed in GPL code."
#endif
#undef INCLUDE_ALLOW_DISTRIBUTE

#if defined VMKERNEL_MODULE && !defined VMKERNEL && \
    !defined INCLUDE_ALLOW_VMK_MODULE && !defined INCLUDE_ALLOW_VMKDRIVERS
#error "The surrounding include file is not allowed in vmkernel modules."
#endif
#undef INCLUDE_ALLOW_VMK_MODULE
#undef INCLUDE_ALLOW_VMKDRIVERS

#if defined INCLUDE_ALLOW_MKS && !(defined COREMKS)
#error "The surrounding include file is not allowed outside of the MKS."
#endif
#undef INCLUDE_ALLOW_MKS
�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_pci.h��������������������������������������������������������������������0000444�0000000�0000000�00000005152�13432725347�015707� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1999 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * compat_pci.h: PCI compatibility wrappers.
 */

#ifndef __COMPAT_PCI_H__
#define __COMPAT_PCI_H__

#include "compat_ioport.h"
#include <linux/pci.h>

#ifndef DMA_BIT_MASK
#  define DMA_BIT_MASK(n) DMA_##n##BIT_MASK
#endif

/*
 * Power Management related compat wrappers.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 10)
#   define compat_pci_save_state(pdev)      pci_save_state((pdev), NULL)
#   define compat_pci_restore_state(pdev)   pci_restore_state((pdev), NULL)
#else
#   define compat_pci_save_state(pdev)      pci_save_state((pdev))
#   define compat_pci_restore_state(pdev)   pci_restore_state((pdev))
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 11)
#   define pm_message_t          u32
#   define compat_pci_choose_state(pdev, state)  (state)
#   define PCI_D0               0
#   define PCI_D3hot            3
#else
#   define compat_pci_choose_state(pdev, state)  pci_choose_state((pdev), (state))
#endif

/* 2.6.14 changed the PCI shutdown callback */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 14)
#   define COMPAT_PCI_SHUTDOWN(func)               .driver = { .shutdown = (func), }
#   define COMPAT_PCI_DECLARE_SHUTDOWN(func, var)  (func)(struct device *(var))
#   define COMPAT_PCI_TO_DEV(dev)                  (to_pci_dev(dev))
#else
#   define COMPAT_PCI_SHUTDOWN(func)               .shutdown = (func)
#   define COMPAT_PCI_DECLARE_SHUTDOWN(func, var)  (func)(struct pci_dev *(var))
#   define COMPAT_PCI_TO_DEV(dev)                  (dev)
#endif

/* 2.6.26 introduced the device_set_wakeup_enable() function */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26)
#   define compat_device_set_wakeup_enable(dev, val) do {} while(0)
#else
#   define compat_device_set_wakeup_enable(dev, val) \
       device_set_wakeup_enable(dev, val)
#endif

#endif /* __COMPAT_PCI_H__ */
����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/kernelStubs.h�������������������������������������������������������������������0000444�0000000�0000000�00000021124�13432725330�016057� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * kernelStubs.h
 *
 * KernelStubs implements some userspace library functions in terms
 * of kernel functions to allow library userspace code to be used in a
 * kernel.
 */

#ifndef __KERNELSTUBS_H__
#define __KERNELSTUBS_H__

#define KRNL_STUBS_DRIVER_TYPE_POSIX  1
#define KRNL_STUBS_DRIVER_TYPE_GDI    2
#define KRNL_STUBS_DRIVER_TYPE_WDM    3
#define KRNL_STUBS_DRIVER_TYPE_NDIS   4

// For now (vsphere-2015), choose a good default. Later we'll modify all the
// build files using KernelStubs to set this.
#ifndef KRNL_STUBS_DRIVER_TYPE
#  if defined(_WIN32)
#     define KRNL_STUBS_DRIVER_TYPE KRNL_STUBS_DRIVER_TYPE_WDM
#  else
#     define KRNL_STUBS_DRIVER_TYPE KRNL_STUBS_DRIVER_TYPE_POSIX
#  endif
#endif

#ifdef linux
#   ifndef __KERNEL__
#      error "__KERNEL__ is not defined"
#   endif
#   include "driver-config.h" // Must be included before any other header files
#   include "vm_basic_types.h"
#   include <linux/kernel.h>
#   include <linux/string.h>
#elif defined(_WIN32)
#   define _CRT_ALLOCATION_DEFINED // prevent malloc.h from defining malloc et. all
#   if KRNL_STUBS_DRIVER_TYPE == KRNL_STUBS_DRIVER_TYPE_GDI
#      include <d3d9.h>
#      include <winddi.h>
#      include <stdio.h>
#      include "vm_basic_types.h"
#      include "vm_basic_defs.h"
#      include "vm_assert.h"
#   elif KRNL_STUBS_DRIVER_TYPE == KRNL_STUBS_DRIVER_TYPE_NDIS
#      include <ntddk.h>
#      include <stdio.h>    /* for _vsnprintf, vsprintf */
#      include <stdarg.h>   /* for va_start stuff */
#      include "vm_basic_defs.h"
#      include "vm_assert.h"
#      include "kernelStubsFloorFixes.h"
#pragma warning(disable:4201) // unnamed struct/union
#      include <ndis.h>
#   elif KRNL_STUBS_DRIVER_TYPE == KRNL_STUBS_DRIVER_TYPE_WDM
#      include "vm_basic_types.h"
#      if defined(NTDDI_WINXP) && (NTDDI_VERSION >= NTDDI_WINXP)
#         include <wdm.h>   /* kernel memory APIs, DbgPrintEx */
#      else
#         include <ntddk.h> /* kernel memory APIs */
#      endif
#      include <stdio.h>    /* for _vsnprintf, vsprintf */
#      include <stdarg.h>   /* for va_start stuff */
#      include <stdlib.h>   /* for min macro. */
#      include "vm_basic_defs.h"
#      include "vm_assert.h"  /* Our assert macros */
#      include "kernelStubsFloorFixes.h"
#   else
#      error Type KRNL_STUBS_DRIVER_TYPE must be defined.
#   endif
#elif defined(__FreeBSD__)
#   include "vm_basic_types.h"
#   ifndef _KERNEL
#      error "_KERNEL is not defined"
#   endif
#   include <sys/types.h>
#   include <sys/malloc.h>
#   include <sys/param.h>
#   include <sys/kernel.h>
#   include <machine/stdarg.h>
#   include <sys/libkern.h>
#elif defined(__APPLE__)
#   include "vm_basic_types.h"
#   ifndef KERNEL
#      error "KERNEL is not defined"
#   endif
#   include <stdarg.h>
#   include <string.h>
# elif defined(sun)
#   include "vm_basic_types.h"
#   include <sys/types.h>
#   include <sys/varargs.h>
#endif
#include "kernelStubsSal.h"

/*
 * Function Prototypes
 */

#if defined(__linux__) || defined(__APPLE__) || defined (sun)

#  ifdef linux                               /* if (linux) { */
char *strdup(const char *source);
#  endif

/* Shared between Linux and Apple kernel stubs. */
void *malloc(size_t size);
void free(void *mem);
void *calloc(size_t num, size_t len);
void *realloc(void *ptr, size_t newSize);

#elif defined(_WIN32)                           /* } else if (_WIN32) { */

_Ret_allocates_malloc_mem_opt_bytecap_(_Size)
_When_windrv_(_IRQL_requires_max_(DISPATCH_LEVEL))
_CRTNOALIAS _CRTRESTRICT
void * __cdecl malloc(
   _In_ size_t _Size);

_Ret_allocates_malloc_mem_opt_bytecount_(_Count*_Size)
_When_windrv_(_IRQL_requires_max_(DISPATCH_LEVEL))
_CRTNOALIAS _CRTRESTRICT
void * __cdecl calloc(
   _In_ size_t _Count,
   _In_ size_t _Size);

_When_windrv_(_IRQL_requires_max_(DISPATCH_LEVEL))
_CRTNOALIAS
void __cdecl free(
   _In_frees_malloc_mem_opt_ void * _Memory);

_Success_(return != 0)
_When_(_Memory != 0, _Ret_reallocates_malloc_mem_opt_newbytecap_oldbytecap_(_NewSize, ((uintptr_t*)_Memory)[-1]))
_When_(_Memory == 0, _Ret_reallocates_malloc_mem_opt_newbytecap_(_NewSize))
_When_windrv_(_IRQL_requires_max_(DISPATCH_LEVEL))
_CRTNOALIAS _CRTRESTRICT
void * __cdecl realloc(
   _In_reallocates_malloc_mem_opt_oldptr_ void * _Memory,
   _In_ size_t _NewSize);

_Success_(return != 0)
_Ret_allocates_malloc_mem_opt_z_
_When_windrv_(_IRQL_requires_max_(DISPATCH_LEVEL))
_CRTIMP
char * __cdecl _strdup_impl(
   _In_opt_z_ const char * _Src);

#define strdup _strdup_impl

#elif defined(__FreeBSD__)                      /* } else if (FreeBSD) { */

/* Kernel memory on FreeBSD is tagged for statistics and sanity checking. */
MALLOC_DECLARE(M_VMWARE_TEMP);

/*
 * On FreeBSD, the general memory allocator for both userland and the kernel is named
 * malloc, but the kernel malloc() takes more arguments.  The following alias & macros
 * work around this, to provide the standard malloc() API for userspace code that is
 * being used in the kernel.
 */

#   undef malloc

static INLINE void *
__compat_malloc(unsigned long size, struct malloc_type *type, int flags) {
   return malloc(size, type, flags);
}

#   define malloc(size)         __compat_malloc(size, M_VMWARE_TEMP, M_NOWAIT)
#   define calloc(count, size)  __compat_malloc((count) * (size),       \
                                                M_VMWARE_TEMP, M_NOWAIT|M_ZERO)
#   define realloc(buf, size)   realloc(buf, size, M_VMWARE_TEMP, M_NOWAIT)
#   define free(buf)            free(buf, M_VMWARE_TEMP)
#   define strchr(s,c)          index(s,c)
#   define strrchr(s,c)         rindex(s,c)

#endif                                          /* } */

_Ret_writes_z_(maxSize)
char *Str_Strcpy(
   _Out_z_cap_(maxSize) char *buf,
   _In_z_ const char *src,
   _In_ size_t maxSize);

_Ret_writes_z_(maxSize)
char *Str_Strcat(
   _Inout_z_cap_(maxSize) char *buf,
   _In_z_ const char *src,
   _In_ size_t maxSize);

_Success_(return >= 0)
int Str_Sprintf(
   _Out_z_cap_(maxSize) _Post_z_count_(return+1) char *buf,
   _In_ size_t maxSize,
   _In_z_ _Printf_format_string_ const char *fmt,
   ...) PRINTF_DECL(3, 4);

_Success_(return != -1)
int Str_Vsnprintf(
   _Out_z_cap_(size) _Post_z_count_(return+1) char *str,
   _In_ size_t size,
   _In_z_ _Printf_format_string_ const char *format,
   _In_ va_list ap) PRINTF_DECL(3, 0);

_Success_(return != 0)
_When_(length != 0, _Ret_allocates_malloc_mem_opt_z_bytecount_(*length))
_When_(length == 0, _Ret_allocates_malloc_mem_opt_z_)
_When_windrv_(_IRQL_requires_max_(DISPATCH_LEVEL))
char *Str_Vasprintf(
   _Out_opt_ size_t *length,
   _In_z_ _Printf_format_string_ const char *format,
   _In_ va_list arguments) PRINTF_DECL(2, 0);

_Success_(return != 0)
_When_(length != 0, _Ret_allocates_malloc_mem_opt_z_bytecount_(*length))
_When_(length == 0, _Ret_allocates_malloc_mem_opt_z_)
_When_windrv_(_IRQL_requires_max_(DISPATCH_LEVEL))
char *Str_Asprintf(
   _Out_opt_ size_t *length,
   _In_z_ _Printf_format_string_ const char *format,
   ...) PRINTF_DECL(2, 3);

#ifdef _WIN32
#pragma warning(push)
#pragma warning(disable: 28301) // Suppress complaint that first declaration lacked annotations
#endif

// For now (vsphere-2015), we don't implement Panic, Warning, or Debug in the
// GDI case.
#if (KRNL_STUBS_DRIVER_TYPE != KRNL_STUBS_DRIVER_TYPE_GDI) &&\
    (KRNL_STUBS_DRIVER_TYPE != KRNL_STUBS_DRIVER_TYPE_NDIS)

/*
 * Stub functions we provide.
 */
#ifdef _WIN32
NORETURN
#endif
void Panic(
   _In_z_ _Printf_format_string_ const char *fmt,
   ...) PRINTF_DECL(1, 2);

void Warning(
   _In_z_ _Printf_format_string_ const char *fmt,
   ...) PRINTF_DECL(1, 2);

/*
 * Functions the driver must implement for the stubs.
 */
EXTERN void Debug(
   _In_z_ _Printf_format_string_ const char *fmt,
   ...) PRINTF_DECL(1, 2);

#endif // KRNL_STUBS_DRIVER_TYPE != KRNL_STUBS_DRIVER_TYPE_GDI

#ifdef _WIN32
#pragma warning(pop)
#endif

#endif /* __KERNELSTUBS_H__ */
��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_log2.h�������������������������������������������������������������������0000444�0000000�0000000�00000003672�13432725347�016004� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2011 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_LOG2_H__
#   define __COMPAT_LOG2_H__

#ifndef LINUX_VERSION_CODE
#   error "Include compat_version.h before compat_log2.h"
#endif

/* linux/log2.h was introduced in 2.6.20. */
#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 19)
#   include <linux/log2.h>
#endif

/*
 * is_power_of_2 was introduced in 2.6.21. This implementation is almost
 * identical to the one found there.
 */
#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 20)
#define compat_is_power_of_2(n) is_power_of_2(n)
#else
static inline __attribute__((const))
int compat_is_power_of_2(unsigned long n)
{
	return (n != 0 && ((n && (n - 1)) == 0));
}
#endif

/*
 * rounddown_power_of_two was introduced in 2.6.24. This implementation is
 * similar to the one in log2.h but with input of int instead of long to
 * avoid more version related checks for fls_long().
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
#define compat_rounddown_pow_of_two(n) rounddown_pow_of_two(n)
#else
static inline __attribute__((const))
unsigned int compat_rounddown_pow_of_two(unsigned int n)
{
	return 1U << (fls(n) -1);
}
#endif

#endif /* __COMPAT_LOG2_H__ */
����������������������������������������������������������������������vmxnet3-only/shared/compat_cred.h�������������������������������������������������������������������0000444�0000000�0000000�00000003244�13432725347�016051� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_CRED_H__
#   define __COMPAT_CRED_H__

/*
 * Include linux/cred.h via linux/sched.h - it is not nice, but
 * as cpp does not have #ifexist...
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)
#include <linux/sched.h>
#else
#include <linux/cred.h>
#endif

#if !defined(current_fsuid) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29)
#define current_uid() (current->uid)
#define current_euid() (current->euid)
#define current_fsuid() (current->fsuid)
#define current_gid() (current->gid)
#define current_egid() (current->egid)
#define current_fsgid() (current->fsgid)
#endif

#if !defined(cap_set_full)
/* cap_set_full was removed in kernel version 3.0-rc4. */
#define cap_set_full(_c) do { (_c) = CAP_FULL_SET; } while (0)
#endif

#if !defined(GLOBAL_ROOT_UID)
#define GLOBAL_ROOT_UID (0)
#endif

#endif /* __COMPAT_CRED_H__ */
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_ioport.h�����������������������������������������������������������������0000444�0000000�0000000�00000004041�13432725347�016444� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2003 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_IOPORT_H__
#   define __COMPAT_IOPORT_H__

#include <linux/ioport.h>

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 0)
static inline void *
compat_request_region(unsigned long start, unsigned long len, const char *name)
{
   if (check_region(start, len)) {
      return NULL;
   }
   request_region(start, len, name);
   return (void*)1;
}
#else
#define compat_request_region(start, len, name) request_region(start, len, name)
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 3, 7)
/* mmap io support starts from 2.3.7, fail the call for kernel prior to that */
static inline void *
compat_request_mem_region(unsigned long start, unsigned long len, const char *name)
{
   return NULL;
}

static inline void
compat_release_mem_region(unsigned long start, unsigned long len)
{
   return;
}
#else
#define compat_request_mem_region(start, len, name) request_mem_region(start, len, name)
#define compat_release_mem_region(start, len)       release_mem_region(start, len)
#endif

/* these two macro defs are needed by compat_pci_request_region */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 3, 15)
#   define IORESOURCE_IO    0x00000100
#   define IORESOURCE_MEM   0x00000200
#endif

#endif /* __COMPAT_IOPORT_H__ */
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vm_atomic.h���������������������������������������������������������������������0000444�0000000�0000000�00000305435�13432725350�015550� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vm_atomic.h --
 *
 *       Atomic power
 *
 * Note: Only partially tested on ARM processors: Works for View Open
 *       Client, which shouldn't have threads, and ARMv8 processors.
 *
 *       In ARM, GCC intrinsics (__sync*) compile but might not
 *       work, while MS intrinsics (_Interlocked*) do not compile.
 */

#ifndef _ATOMIC_H_
#define _ATOMIC_H_

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_VMKDRIVERS
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMCORE
#include "includeCheck.h"

#include "vm_basic_types.h"
#include "vm_assert.h"

#if defined __cplusplus
extern "C" {
#endif

/*
 * There are two concepts involved when dealing with atomic accesses:
 * 1. Atomicity of the access itself
 * 2. Ordering of the access with respect to other reads&writes (from the view
 *    of other processors/devices).
 *
 * Two examples help to clarify #2:
 * a. Inc: A caller implementing a simple independent global event counter
 *         might not care if the compiler or processor visibly reorders the
 *         increment around other memory accesses.
 * b. Dec: A caller implementing a reference count absolutely *doesn't* want
 *         the compiler or processor to visibly reordering writes after that
 *         decrement: if that happened, the program could then end up writing
 *         to memory that was freed by another processor.
 *
 * C11 has standardized a good model for expressing these orderings when doing
 * atomics. It defines three *tiers* of ordering:
 * 1. Sequential Consistency (every processor sees the same total order of
 *    events)
 *
 * 2. Acquire/Release ordering (roughly, everybody can agree previous events
 *    have completed, but they might disagree on the ordering of previous
 *    independent events).
 *
 *    The relative ordering provided by this tier is sufficient for common
 *    locking and initialization activities, but is insufficient for unusual
 *    synchronization schemes (e.g. IRIW aka Independent Read Independent
 *    Write designs such Dekker's algorithm, Peterson's algorithm, etc.)
 *
 *    In other words, this tier is close in behavior to Sequential Consistency
 *    in much the same way a General-Relativity universe is close to a
 *    Newtonian universe.
 * 3. Relaxed (i.e unordered/unfenced)
 *
 * In C11 standard's terminology for atomic memory ordering,
 * - in case (a) we want "relaxed" ordering for perf and,
 * - in case (b) we want "sequentially consistent" ordering (or perhaps the
 *   only slightly weaker "release" ordering) for correctness.
 *
 * There are standardized mappings of operations to orderings for every
 * processor architecture. See
 * - https://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
 * - http://preshing.com/20120913/acquire-and-release-semantics/
 *
 * In this file:
 * 1. all RMW (Read/Modify/Write) operations are sequentially consistent.
 *    This includes operations like Atomic_IncN, Atomic_ReadIfEqualWriteN,
 *    Atomic_ReadWriteN, etc.
 * 2. all R and W operations are relaxed. This includes operations like
 *    Atomic_WriteN, Atomic_ReadN, Atomic_TestBitN, etc.
 *
 * The below routines of course ensure both the CPU and compiler honor the
 * ordering constraint.
 *
 * Notes:
 * 1. Since R-only and W-only operations do not provide ordering, callers
 *    using them for synchronizing operations like double-checked
 *    initialization or releasing spinlocks must provide extra barriers.
 * 2. This implementation of Atomic operations is suboptimal. On x86,simple
 *    reads and writes have acquire/release semantics at the hardware level.
 *    On arm64, we have separate instructions for sequentially consistent
 *    reads and writes (the same instructions are used for acquire/release).
 *    Neither of these are exposed for R-only or W-only callers.
 *
 * For further details on x86 and ARM memory ordering see
 * https://wiki.eng.vmware.com/ARM/MemoryOrdering.
 */

#ifdef VM_ARM_64
#   include "vm_atomic_arm64_begin.h"
#endif

/* Basic atomic types: 8, 16, 32, 64 and 128 bits */
typedef struct Atomic_uint8 {
   volatile uint8 value;
} Atomic_uint8 ALIGNED(1);

typedef struct Atomic_uint16 {
   volatile uint16 value;
} Atomic_uint16 ALIGNED(2);

typedef struct Atomic_uint32 {
   volatile uint32 value;
} Atomic_uint32 ALIGNED(4);

typedef struct Atomic_uint64 {
   volatile uint64 value;
} Atomic_uint64 ALIGNED(8);

#if defined __GNUC__ && defined VM_64BIT && \
     (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64)
typedef struct Atomic_uint128 {
   volatile uint128 value;
} Atomic_uint128 ALIGNED(16);
#endif

/*
 * Prototypes for msft atomics.  These are defined & inlined by the
 * compiler so no function definition is needed.  The prototypes are
 * needed for C++.
 *
 * The declarations for the intrinsic functions were taken from ntddk.h
 * in the DDK. The declarations must match otherwise the 64-bit C++
 * compiler will complain about second linkage of the intrinsic functions.
 * We define the intrinsic using the basic types corresponding to the
 * Windows typedefs. This avoids having to include windows header files
 * to get to the windows types.
 */
#if defined _MSC_VER && !defined BORA_NO_WIN32_INTRINS
#ifdef __cplusplus
extern "C" {
#endif
long  _InterlockedExchange(long volatile*, long);
long  _InterlockedCompareExchange(long volatile*, long, long);
long  _InterlockedExchangeAdd(long volatile*, long);
long  _InterlockedDecrement(long volatile*);
long  _InterlockedIncrement(long volatile*);
__int64  _InterlockedCompareExchange64(__int64 volatile*, __int64, __int64);
#pragma intrinsic(_InterlockedExchange, _InterlockedCompareExchange)
#pragma intrinsic(_InterlockedExchangeAdd, _InterlockedDecrement)
#pragma intrinsic(_InterlockedIncrement)
#pragma intrinsic(_InterlockedCompareExchange64)

# if _MSC_VER >= 1600
char     _InterlockedExchange8(char volatile *, char);
char     _InterlockedCompareExchange8(char volatile *, char, char);
#pragma intrinsic(_InterlockedCompareExchange8, _InterlockedCompareExchange8)
#endif

#if defined VM_X86_64
long     _InterlockedAnd(long volatile*, long);
__int64  _InterlockedAnd64(__int64 volatile*, __int64);
long     _InterlockedOr(long volatile*, long);
__int64  _InterlockedOr64(__int64 volatile*, __int64);
long     _InterlockedXor(long volatile*, long);
__int64  _InterlockedXor64(__int64 volatile*, __int64);
__int64  _InterlockedExchangeAdd64(__int64 volatile*, __int64);
__int64  _InterlockedIncrement64(__int64 volatile*);
__int64  _InterlockedDecrement64(__int64 volatile*);
__int64  _InterlockedExchange64(__int64 volatile*, __int64);
#if !defined _WIN64
#pragma intrinsic(_InterlockedAnd, _InterlockedAnd64)
#pragma intrinsic(_InterlockedOr, _InterlockedOr64)
#pragma intrinsic(_InterlockedXor, _InterlockedXor64)
#pragma intrinsic(_InterlockedExchangeAdd64, _InterlockedIncrement64)
#pragma intrinsic(_InterlockedDecrement64, _InterlockedExchange64)
#endif /* !_WIN64 */
#endif /* __x86_64__ */

#ifdef __cplusplus
}
#endif
#endif /* _MSC_VER */

#if defined __arm__
/*
 * LDREX without STREX or CLREX may cause problems in environments where the
 * context switch may not clear the reference monitor - according ARM manual
 * the reference monitor should be cleared after a context switch, but some
 * may not like Linux kernel's non-preemptive context switch path. So use of
 * ARM routines in kernel code may not be safe.
 */
#   if defined __ARM_ARCH_7__ || defined __ARM_ARCH_7A__ ||  \
       defined __ARM_ARCH_7R__|| defined __ARM_ARCH_7M__
#      define VM_ARM_V7
#      ifdef __KERNEL__
#         warning LDREX/STREX may not be safe in linux kernel, since it      \
          does not issue CLREX on context switch (as of 2011-09-29).
#      endif
#   else
#     error Only ARMv7 extends the synchronization primitives ldrex/strex.   \
            For the lower ARM version, please implement the atomic functions \
            by kernel APIs.
#   endif
#endif

/* Data Memory Barrier */
#ifdef VM_ARM_V7
#define dmb() __asm__ __volatile__("dmb" : : : "memory")
#endif

/* Convert a volatile uint32 to Atomic_uint32. */
static INLINE Atomic_uint32 *
Atomic_VolatileToAtomic32(volatile uint32 *var)  // IN:
{
   return (Atomic_uint32 *)var;
}
#define Atomic_VolatileToAtomic Atomic_VolatileToAtomic32

/* Convert a volatile uint64 to Atomic_uint64. */
static INLINE Atomic_uint64 *
Atomic_VolatileToAtomic64(volatile uint64 *var)  // IN:
{
   return (Atomic_uint64 *)var;
}

/*
 * The Read/Modify/Write operations on x86/x64 are all written using the
 * "memory" constraint. This is to ensure the compiler treats the operation as
 * a full barrier, flushing any pending/cached state currently residing in
 * registers.
 */

#if defined _MSC_VER && _MSC_VER < 1600 && defined __x86_64__
uint8 VMWInterlockedExchange8(uint8 volatile *ptr,
                              uint8 val);

uint8 VMWInterlockedCompareExchange8(uint8 volatile *ptr,
                                     uint8 newVal,
                                     uint8 oldVal);
#endif

#if defined __GNUC__ && defined VM_ARM_32
/* Force the link step to fail for unimplemented functions. */
extern int AtomicUndefined(void const *);
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadIfEqualWrite128 --
 *
 *      Compare and exchange a 16 byte tuple.
 *
 * Results:
 *      old value
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */
#if defined __GNUC__ && defined VM_64BIT && \
     (defined __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 || defined VM_ARM_64)
static INLINE uint128
Atomic_ReadIfEqualWrite128(Atomic_uint128 *ptr,   // IN/OUT
                           uint128        oldVal, // IN
                           uint128        newVal) // IN
{
#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16
   return __sync_val_compare_and_swap(&ptr->value, oldVal, newVal);
#elif defined VM_ARM_64
   union {
      uint128 raw;
      struct {
         uint64 lo;
         uint64 hi;
      };
   } res, _old = { oldVal }, _new = { newVal };
   uint32 failed;

SMP_RW_BARRIER_RW();
   __asm__ __volatile__(
      "1: ldxp    %x0, %x1, %3        \n\t"
      "   cmp     %x0, %x4            \n\t"
      "   ccmp    %x1, %x5, #0, eq    \n\t"
      "   b.ne    2f                  \n\t"
      "   stxp    %w2, %x6, %x7, %3   \n\t"
      "   cbnz    %w2, 1b             \n\t"
      "2:                             \n\t"
      : "=&r" (res.lo),
        "=&r" (res.hi),
        "=&r" (failed),
        "+Q" (ptr->value)
      : "r" (_old.lo),
        "r" (_old.hi),
        "r" (_new.lo),
        "r" (_new.hi)
      : "cc"
   );
   SMP_RW_BARRIER_RW();

return res.raw;
#endif
}
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Read8 --
 *
 *      Read the value of the specified object atomically.
 *
 * Results:
 *      The value of the atomic variable.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint8
Atomic_Read8(Atomic_uint8 const *var)  // IN:
{
   uint8 val;

#if defined __GNUC__ && defined VM_ARM_32
   val = AtomicUndefined(var);
#elif defined VM_ARM_64
   val = _VMATOM_X(R, 8, &var->value);
#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__)
   __asm__ __volatile__(
      "movb %1, %0"
      : "=q" (val)
      : "m" (var->value)
   );
#elif defined _MSC_VER
   val = var->value;
#else
#error No compiler defined for Atomic_Read8
#endif

return val;
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadWrite8 --
 *
 *      Read followed by write.
 *
 * Results:
 *      The value of the atomic variable before the write.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint8
Atomic_ReadWrite8(Atomic_uint8 *var,  // IN/OUT:
                  uint8 val)          // IN:
{
#if defined __GNUC__ && defined VM_ARM_32
   return AtomicUndefined(var + val);
#elif defined VM_ARM_64
   return _VMATOM_X(RW, 8, TRUE, &var->value, val);
#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__)
   __asm__ __volatile__(
      "xchgb %0, %1"
      : "=q" (val),
        "+m" (var->value)
      : "0" (val)
      : "memory"
   );
   return val;
#elif defined _MSC_VER && _MSC_VER >= 1600
   return _InterlockedExchange8((volatile char *)&var->value, val);
#elif defined _MSC_VER && defined __i386__
#pragma warning(push)
#pragma warning(disable : 4035)         // disable no-return warning
   {
      __asm movzx eax, val
      __asm mov ebx, var
      __asm xchg [ebx]Atomic_uint8.value, al
   }
#pragma warning(pop)
#elif defined _MSC_VER && defined __x86_64__
   return VMWInterlockedExchange8(&var->value, val);
#else
#error No compiler defined for Atomic_ReadWrite8
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Write8 --
 *
 *      Write the specified value to the specified object atomically.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Write8(Atomic_uint8 *var,  // IN/OUT:
              uint8 val)          // IN:
{
#if defined __GNUC__ && defined VM_ARM_32
   AtomicUndefined(var + val);
#elif defined VM_ARM_64
   _VMATOM_X(W, 8, &var->value, val);
#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__)
   __asm__ __volatile__(
      "movb %1, %0"
      : "=m" (var->value)
      : "qn" (val)
   );
#elif defined _MSC_VER
   var->value = val;
#else
#error No compiler defined for Atomic_Write8
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadIfEqualWrite8 --
 *
 *      Compare exchange: Read variable, if equal to oldVal, write newVal.
 *
 * Results:
 *      The value of the atomic variable before the write.
 *
 * Side effects:
 *      The variable may be modified.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint8
Atomic_ReadIfEqualWrite8(Atomic_uint8 *var,  // IN/OUT:
                         uint8 oldVal,       // IN:
                         uint8 newVal)       // IN:
{
#if defined __GNUC__ && defined VM_ARM_32
   return AtomicUndefined(var + oldVal + newVal);
#elif defined VM_ARM_64
   return _VMATOM_X(RIFEQW, 8, TRUE, &var->value, oldVal, newVal);
#elif defined __GNUC__ && (defined __i386__ || defined __x86_64__)
   uint8 val;

__asm__ __volatile__(
      "lock; cmpxchgb %2, %1"
      : "=a" (val),
        "+m" (var->value)
      : "q" (newVal),
        "0" (oldVal)
      : "cc", "memory"
   );

return val;
#elif defined _MSC_VER && _MSC_VER >= 1600
   return _InterlockedCompareExchange8((volatile char *)&var->value,
                                       newVal, oldVal);
#elif defined _MSC_VER && defined __i386__
#pragma warning(push)
#pragma warning(disable : 4035)         // disable no-return warning
   {
      __asm mov al, oldVal
      __asm mov ebx, var
      __asm mov cl, newVal
      __asm lock cmpxchg [ebx]Atomic_uint8.value, cl
      __asm movzx eax, al
      // eax is the return value, this is documented to work - edward
   }
#pragma warning(pop)
#elif defined _MSC_VER && defined __x86_64__
   return VMWInterlockedCompareExchange8(&var->value, newVal, oldVal);
#else
#error No compiler defined for Atomic_ReadIfEqualWrite8
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadAnd8 --
 *
 *      Atomic read (returned), bitwise AND with a value, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint8
Atomic_ReadAnd8(Atomic_uint8 *var, // IN/OUT
                uint8 val)         // IN
{
   uint8 res;

#if defined VM_ARM_64
   res = _VMATOM_X(ROP, 8, TRUE, &var->value, and, val);
#else
   do {
      res = Atomic_Read8(var);
   } while (res != Atomic_ReadIfEqualWrite8(var, res, res & val));
#endif

return res;
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_And8 --
 *
 *      Atomic read, bitwise AND with a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_And8(Atomic_uint8 *var, // IN/OUT
            uint8 val)         // IN
{
#if defined VM_ARM_64
   _VMATOM_X(OP, 8, TRUE, &var->value, and, val);
#else
   (void)Atomic_ReadAnd8(var, val);
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadOr8 --
 *
 *      Atomic read (returned), bitwise OR with a value, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint8
Atomic_ReadOr8(Atomic_uint8 *var, // IN/OUT
               uint8 val)         // IN
{
   uint8 res;

#if defined VM_ARM_64
   res = _VMATOM_X(ROP, 8, TRUE, &var->value, orr, val);
#else
   do {
      res = Atomic_Read8(var);
   } while (res != Atomic_ReadIfEqualWrite8(var, res, res | val));
#endif

return res;
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Or8 --
 *
 *      Atomic read, bitwise OR with a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Or8(Atomic_uint8 *var, // IN/OUT
           uint8 val)         // IN
{
#if defined VM_ARM_64
   _VMATOM_X(OP, 8, TRUE, &var->value, orr, val);
#else
   (void)Atomic_ReadOr8(var, val);
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadXor8 --
 *
 *      Atomic read (returned), bitwise XOR with a value, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint8
Atomic_ReadXor8(Atomic_uint8 *var, // IN/OUT
                uint8 val)         // IN
{
   uint8 res;

#if defined VM_ARM_64
   res = _VMATOM_X(ROP, 8, TRUE, &var->value, eor, val);
#else
   do {
      res = Atomic_Read8(var);
   } while (res != Atomic_ReadIfEqualWrite8(var, res, res ^ val));
#endif

return res;
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Xor8 --
 *
 *      Atomic read, bitwise XOR with a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Xor8(Atomic_uint8 *var, // IN/OUT
            uint8 val)         // IN
{
#if defined VM_ARM_64
   _VMATOM_X(OP, 8, TRUE, &var->value, eor, val);
#else
   (void)Atomic_ReadXor8(var, val);
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadAdd8 --
 *
 *      Atomic read (returned), add a value, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint8
Atomic_ReadAdd8(Atomic_uint8 *var, // IN/OUT
                uint8 val)         // IN
{
   uint8 res;

#if defined VM_ARM_64
   res = _VMATOM_X(ROP, 8, TRUE, &var->value, add, val);
#else
   do {
      res = Atomic_Read8(var);
   } while (res != Atomic_ReadIfEqualWrite8(var, res, res + val));
#endif

return res;
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Add8 --
 *
 *      Atomic read, add a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Add8(Atomic_uint8 *var, // IN/OUT
            uint8 val)         // IN
{
#if defined VM_ARM_64
   _VMATOM_X(OP, 8, TRUE, &var->value, add, val);
#else
   (void)Atomic_ReadAdd8(var, val);
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Sub8 --
 *
 *      Atomic read, subtract a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Sub8(Atomic_uint8 *var, // IN/OUT
            uint8 val)         // IN
{
#if defined VM_ARM_64
   _VMATOM_X(OP, 8, TRUE, &var->value, sub, val);
#else
   Atomic_Add8(var, -val);
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Inc8 --
 *
 *      Atomic read, increment, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Inc8(Atomic_uint8 *var) // IN/OUT
{
   Atomic_Add8(var, 1);
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Dec8 --
 *
 *      Atomic read, decrement, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Dec8(Atomic_uint8 *var) // IN/OUT
{
   Atomic_Sub8(var, 1);
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadInc8 --
 *
 *      Atomic read (returned), increment, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint8
Atomic_ReadInc8(Atomic_uint8 *var) // IN/OUT
{
   return Atomic_ReadAdd8(var, 1);
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadDec8 --
 *
 *      Atomic read (returned), decrement, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint8
Atomic_ReadDec8(Atomic_uint8 *var) // IN/OUT
{
   return Atomic_ReadAdd8(var, (uint8)-1);
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Read32 --
 *
 *      Read
 *
 * Results:
 *      The value of the atomic variable.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint32
Atomic_Read32(Atomic_uint32 const *var) // IN
{
   uint32 value;

#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE
   ASSERT(((uintptr_t)var % 4) == 0);
#endif

#if defined __GNUC__
   /*
    * Use inline assembler to force using a single load instruction to
    * ensure that the compiler doesn't split a transfer operation into multiple
    * instructions.
    */

#if defined VM_ARM_32
   __asm__ __volatile__(
      "ldr %0, [%1]"
      : "=r" (value)
      : "r" (&var->value)
   );
#elif defined VM_ARM_64
   value = _VMATOM_X(R, 32, &var->value);
#else
   __asm__ __volatile__(
      "mov %1, %0"
      : "=r" (value)
      : "m" (var->value)
   );
#endif
#elif defined _MSC_VER
   /*
    * Microsoft docs guarantee simple reads and writes to properly
    * aligned 32-bit variables use only a single instruction.
    * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx
    */

value = var->value;
#else
#error No compiler defined for Atomic_Read
#endif

return value;
}
#define Atomic_Read Atomic_Read32

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadWrite32 --
 *
 *      Read followed by write
 *
 * Results:
 *      The value of the atomic variable before the write.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint32
Atomic_ReadWrite32(Atomic_uint32 *var, // IN/OUT
                   uint32 val)         // IN
{
#if defined __GNUC__
#ifdef VM_ARM_V7
   register volatile uint32 retVal;
   register volatile uint32 res;

dmb();

__asm__ __volatile__(
   "1: ldrex %[retVal], [%[var]] \n\t"
      "strex %[res], %[val], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [retVal] "=&r" (retVal), [res] "=&r" (res)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();

return retVal;
#elif defined VM_ARM_64
   return _VMATOM_X(RW, 32, TRUE, &var->value, val);
#else /* VM_X86_ANY */
   /* Checked against the Intel manual and GCC --walken */
   __asm__ __volatile__(
      "xchgl %0, %1"
      : "=r" (val),
        "+m" (var->value)
      : "0" (val)
      : "memory"
   );
   return val;
#endif /* VM_X86_ANY */
#elif defined _MSC_VER
   return _InterlockedExchange((long *)&var->value, (long)val);
#else
#error No compiler defined for Atomic_ReadWrite
#endif // __GNUC__
}
#define Atomic_ReadWrite Atomic_ReadWrite32

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Write32 --
 *
 *      Write
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Write32(Atomic_uint32 *var, // OUT
               uint32 val)         // IN
{
#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE
   ASSERT(((uintptr_t)var % 4) == 0);
#endif

#if defined __GNUC__
#if defined VM_ARM_64
   _VMATOM_X(W, 32, &var->value, val);
#elif defined VM_ARM_32
   /*
    * Best left this way due to the intricacies of exclusive load/store
    * operations on legacy (32-bit) ARM.
    *
    * A3.4.1 ARM DDI 0406C:
    *
    * When a processor writes using any instruction other than a
    * Store-Exclusive:
    *
    * - if the write is to a physical address that is not covered by its local
    *   monitor the write does not affect the state of the local monitor
    * - if the write is to a physical address that is covered by its local
    *   monitor it is IMPLEMENTATION DEFINED whether the write affects the
    *   state of the local monitor.
    *
    * A3.4.5 ARM DDI 0406C:
    *
    * If two STREX instructions are executed without an intervening LDREX the
    * second STREX returns a status value of 1. This means that:
    *
    * - ARM recommends that, in a given thread of execution, every STREX has a
    *   preceding LDREX associated with it
    * - it is not necessary for every LDREX to have a subsequent STREX.
    */

Atomic_ReadWrite32(var, val);
#else
   /*
    * Use inline assembler to force using a single store instruction to
    * ensure that the compiler doesn't split a transfer operation into multiple
    * instructions.
    */

__asm__ __volatile__(
      "mov %1, %0"
      : "=m" (var->value)
      : "r" (val)
   );
#endif
#elif defined _MSC_VER
   /*
    * Microsoft docs guarantee simple reads and writes to properly
    * aligned 32-bit variables use only a single instruction.
    * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx
    */

var->value = val;
#else
#error No compiler defined for Atomic_Write
#endif
}
#define Atomic_Write Atomic_Write32

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadIfEqualWrite32 --
 *
 *      Compare exchange: Read variable, if equal to oldVal, write newVal
 *
 * Results:
 *      The value of the atomic variable before the write.
 *
 * Side effects:
 *      The variable may be modified.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint32
Atomic_ReadIfEqualWrite32(Atomic_uint32 *var, // IN/OUT
                          uint32 oldVal,      // IN
                          uint32 newVal)      // IN
{
#if defined __GNUC__
#ifdef VM_ARM_V7
   register uint32 retVal;
   register uint32 res;

dmb();

__asm__ __volatile__(
   "1: ldrex %[retVal], [%[var]] \n\t"
      "mov %[res], #0 \n\t"
      "teq %[retVal], %[oldVal] \n\t"
      "strexeq %[res], %[newVal], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [retVal] "=&r" (retVal), [res] "=&r" (res)
      : [var] "r" (&var->value), [oldVal] "r" (oldVal), [newVal] "r" (newVal)
      : "cc"
   );

dmb();

return retVal;
#elif defined VM_ARM_64
   return _VMATOM_X(RIFEQW, 32, TRUE, &var->value, oldVal, newVal);
#else /* VM_X86_ANY */
   uint32 val;

/* Checked against the Intel manual and GCC --walken */
   __asm__ __volatile__(
      "lock; cmpxchgl %2, %1"
      : "=a" (val),
        "+m" (var->value)
      : "r" (newVal),
        "0" (oldVal)
      : "cc", "memory"
   );
   return val;
#endif /* VM_X86_ANY */
#elif defined _MSC_VER
   return _InterlockedCompareExchange((long *)&var->value,
                                      (long)newVal,
                                      (long)oldVal);
#else
#error No compiler defined for Atomic_ReadIfEqualWrite
#endif
}
#define Atomic_ReadIfEqualWrite Atomic_ReadIfEqualWrite32

#if defined VM_64BIT || defined VM_ARM_V7
/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadIfEqualWrite64 --
 *
 *      Compare exchange: Read variable, if equal to oldVal, write newVal
 *
 * Results:
 *      The value of the atomic variable before the write.
 *
 * Side effects:
 *      The variable may be modified.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint64
Atomic_ReadIfEqualWrite64(Atomic_uint64 *var, // IN/OUT
                          uint64 oldVal,      // IN
                          uint64 newVal)      // IN
{
#if defined __GNUC__
#ifdef VM_ARM_V7
   register uint64 retVal;
   register uint32 res;

dmb();

/*
    * Under Apple LLVM version 5.0 (clang-500.2.76) (based on LLVM 3.3svn)
    * There will be a warning:
    * "value size does not match register size specified by the constraint
    * and modifier [-Wasm-operand-widths]"
    * on the lines:
    * : [var] "r" (&var->value), [oldVal] "r" (oldVal), [newVal] "r" (newVal)
    *                                          ^
    * : [var] "r" (&var->value), [oldVal] "r" (oldVal), [newVal] "r" (newVal)
    *                                                                 ^
    *
    * Furthermore, using a 32-bits register to store a
    * 64-bits value of an variable looks risky.
    */
#if defined __APPLE__ && __clang__ == 1 && __clang_major__ >= 5
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wasm-operand-widths"
#endif
   __asm__ __volatile__(
   "1: ldrexd %[retVal], %H[retVal], [%[var]] \n\t"
      "mov %[res], #0 \n\t"
      "teq %[retVal], %[oldVal] \n\t"
      "teqeq %H[retVal], %H[oldVal] \n\t"
      "strexdeq %[res], %[newVal], %H[newVal], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [retVal] "=&r" (retVal), [res] "=&r" (res)
      : [var] "r" (&var->value), [oldVal] "r" (oldVal), [newVal] "r" (newVal)
      : "cc"
   );
#if defined __APPLE__ && __clang__ == 1 && __clang_major__ >= 5
#pragma clang diagnostic pop
#endif // defined __APPLE__ && __clang__ == 1 && __clang_major__ >= 5
   dmb();

return retVal;
#elif defined VM_ARM_64
   return _VMATOM_X(RIFEQW, 64, TRUE, &var->value, oldVal, newVal);
#else /* VM_X86_64 */
   uint64 val;

/* Checked against the AMD manual and GCC --hpreg */
   __asm__ __volatile__(
      "lock; cmpxchgq %2, %1"
      : "=a" (val),
        "+m" (var->value)
      : "r" (newVal),
        "0" (oldVal)
      : "cc", "memory"
   );
   return val;
#endif //VM_ARM_V7
#elif defined _MSC_VER
   return _InterlockedCompareExchange64((__int64 *)&var->value,
                                        (__int64)newVal,
                                        (__int64)oldVal);
#else
#error No compiler defined for Atomic_ReadIfEqualWrite64
#endif
}
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_And32 --
 *
 *      Atomic read, bitwise AND with a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_And32(Atomic_uint32 *var, // IN/OUT
             uint32 val)         // IN
{
#if defined __GNUC__
#ifdef VM_ARM_V7
   register volatile uint32 res;
   register volatile uint32 tmp;

dmb();

__asm__ __volatile__(
   "1: ldrex %[tmp], [%[var]] \n\t"
      "and %[tmp], %[tmp], %[val] \n\t"
      "strex %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [res] "=&r" (res), [tmp] "=&r" (tmp)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();
#elif defined VM_ARM_64
   _VMATOM_X(OP, 32, TRUE, &var->value, and, val);
#else /* VM_X86_ANY */
   /* Checked against the Intel manual and GCC --walken */
   __asm__ __volatile__(
      "lock; andl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
      : "cc", "memory"
   );
#endif /* VM_X86_ANY */
#elif defined _MSC_VER
#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedAnd((long *)&var->value, (long)val);
#else
   __asm mov eax, val
   __asm mov ebx, var
   __asm lock and [ebx]Atomic_uint32.value, eax
#endif
#else
#error No compiler defined for Atomic_And
#endif
}
#define Atomic_And Atomic_And32

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Or32 --
 *
 *      Atomic read, bitwise OR with a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Or32(Atomic_uint32 *var, // IN/OUT
            uint32 val)         // IN
{
#if defined __GNUC__
#ifdef VM_ARM_V7
   register volatile uint32 res;
   register volatile uint32 tmp;

dmb();

__asm__ __volatile__(
   "1: ldrex %[tmp], [%[var]] \n\t"
      "orr %[tmp], %[tmp], %[val] \n\t"
      "strex %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [res] "=&r" (res), [tmp] "=&r" (tmp)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();
#elif defined VM_ARM_64
   _VMATOM_X(OP, 32, TRUE, &var->value, orr, val);
#else /* VM_X86_ANY */
   /* Checked against the Intel manual and GCC --walken */
   __asm__ __volatile__(
      "lock; orl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
      : "cc", "memory"
   );
#endif /* VM_X86_ANY */
#elif defined _MSC_VER
#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedOr((long *)&var->value, (long)val);
#else
   __asm mov eax, val
   __asm mov ebx, var
   __asm lock or [ebx]Atomic_uint32.value, eax
#endif
#else
#error No compiler defined for Atomic_Or
#endif
}
#define Atomic_Or Atomic_Or32

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Xor32 --
 *
 *      Atomic read, bitwise XOR with a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Xor32(Atomic_uint32 *var, // IN/OUT
             uint32 val)         // IN
{
#if defined __GNUC__
#ifdef VM_ARM_V7
   register volatile uint32 res;
   register volatile uint32 tmp;

dmb();

__asm__ __volatile__(
   "1: ldrex %[tmp], [%[var]] \n\t"
      "eor %[tmp], %[tmp], %[val] \n\t"
      "strex %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [res] "=&r" (res), [tmp] "=&r" (tmp)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();
#elif defined VM_ARM_64
   _VMATOM_X(OP, 32, TRUE, &var->value, eor, val);
#else /* VM_X86_ANY */
   /* Checked against the Intel manual and GCC --walken */
   __asm__ __volatile__(
      "lock; xorl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
      : "cc", "memory"
   );
#endif /* VM_X86_ANY */
#elif defined _MSC_VER
#if defined __x86_64__ || defined VM_ARM_32
   _InterlockedXor((long *)&var->value, (long)val);
#else
   __asm mov eax, val
   __asm mov ebx, var
   __asm lock xor [ebx]Atomic_uint32.value, eax
#endif
#else
#error No compiler defined for Atomic_Xor
#endif
}
#define Atomic_Xor Atomic_Xor32

#if defined VM_64BIT
/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Xor64 --
 *
 *      Atomic read, bitwise XOR with a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Xor64(Atomic_uint64 *var, // IN/OUT
             uint64 val)         // IN
{
#if defined __GNUC__
#if defined VM_ARM_64
   _VMATOM_X(OP, 64, TRUE, &var->value, eor, val);
#else /* VM_X86_64 */
   /* Checked against the AMD manual and GCC --hpreg */
   __asm__ __volatile__(
      "lock; xorq %1, %0"
      : "+m" (var->value)
      : "re" (val)
      : "cc", "memory"
   );
#endif
#elif defined _MSC_VER
   _InterlockedXor64((__int64 *)&var->value, (__int64)val);
#else
#error No compiler defined for Atomic_Xor64
#endif
}
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Add32 --
 *
 *      Atomic read, add a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Add32(Atomic_uint32 *var, // IN/OUT
             uint32 val)         // IN
{
#if defined __GNUC__
#ifdef VM_ARM_V7
   register volatile uint32 res;
   register volatile uint32 tmp;

dmb();

__asm__ __volatile__(
   "1: ldrex %[tmp], [%[var]] \n\t"
      "add %[tmp], %[tmp], %[val] \n\t"
      "strex %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [res] "=&r" (res), [tmp] "=&r" (tmp)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();
#elif defined VM_ARM_64
   _VMATOM_X(OP, 32, TRUE, &var->value, add, val);
#else /* VM_X86_ANY */
   /* Checked against the Intel manual and GCC --walken */
   __asm__ __volatile__(
      "lock; addl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
      : "cc", "memory"
   );
#endif /* VM_X86_ANY */
#elif defined _MSC_VER
   _InterlockedExchangeAdd((long *)&var->value, (long)val);
#else
#error No compiler defined for Atomic_Add
#endif
}
#define Atomic_Add Atomic_Add32

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Sub32 --
 *
 *      Atomic read, subtract a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Sub32(Atomic_uint32 *var, // IN/OUT
             uint32 val)         // IN
{
#if defined __GNUC__
#ifdef VM_ARM_V7
   register volatile uint32 res;
   register volatile uint32 tmp;

dmb();

__asm__ __volatile__(
      "1: ldrex %[tmp], [%[var]] \n\t"
      "sub %[tmp], %[tmp], %[val] \n\t"
      "strex %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [res] "=&r" (res), [tmp] "=&r" (tmp)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();
#elif defined VM_ARM_64
   _VMATOM_X(OP, 32, TRUE, &var->value, sub, val);
#else /* VM_X86_ANY */
   /* Checked against the Intel manual and GCC --walken */
   __asm__ __volatile__(
      "lock; subl %1, %0"
      : "+m" (var->value)
      : "ri" (val)
      : "cc", "memory"
   );
#endif /* VM_X86_ANY */
#elif defined _MSC_VER
   _InterlockedExchangeAdd((long *)&var->value, -(long)val);
#else
#error No compiler defined for Atomic_Sub
#endif
}
#define Atomic_Sub Atomic_Sub32

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Inc32 --
 *
 *      Atomic read, increment, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Inc32(Atomic_uint32 *var) // IN/OUT
{
#ifdef __GNUC__
#if defined VM_ARM_ANY
   Atomic_Add32(var, 1);
#else /* VM_X86_ANY */
   /* Checked against the Intel manual and GCC --walken */
   __asm__ __volatile__(
      "lock; incl %0"
      : "+m" (var->value)
      :
      : "cc", "memory"
   );
#endif /* VM_X86_ANY */
#elif defined _MSC_VER
   _InterlockedIncrement((long *)&var->value);
#else
#error No compiler defined for Atomic_Inc
#endif
}
#define Atomic_Inc Atomic_Inc32

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Dec32 --
 *
 *      Atomic read, decrement, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Dec32(Atomic_uint32 *var) // IN/OUT
{
#ifdef __GNUC__
#if defined VM_ARM_ANY
   Atomic_Sub32(var, 1);
#else /* VM_X86_ANY */
   /* Checked against the Intel manual and GCC --walken */
   __asm__ __volatile__(
      "lock; decl %0"
      : "+m" (var->value)
      :
      : "cc", "memory"
   );
#endif /* VM_X86_ANY */
#elif defined _MSC_VER
   _InterlockedDecrement((long *)&var->value);
#else
#error No compiler defined for Atomic_Dec
#endif
}
#define Atomic_Dec Atomic_Dec32

/*
 * Note that the technique below can be used to implement ReadX(), where X is
 * an arbitrary mathematical function.
 */

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadOr32 --
 *
 *      Atomic read (returned), bitwise OR with a value, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint32
Atomic_ReadOr32(Atomic_uint32 *var, // IN/OUT
                uint32 val)         // IN
{
   uint32 res;

#if defined VM_ARM_64
   res = _VMATOM_X(ROP, 32, TRUE, &var->value, orr, val);
#else
   do {
      res = Atomic_Read32(var);
   } while (res != Atomic_ReadIfEqualWrite32(var, res, res | val));
#endif

return res;
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadAnd32 --
 *
 *      Atomic read (returned), bitwise And with a value, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint32
Atomic_ReadAnd32(Atomic_uint32 *var, // IN/OUT
                 uint32 val)         // IN
{
   uint32 res;

#if defined VM_ARM_64
   res = _VMATOM_X(ROP, 32, TRUE, &var->value, and, val);
#else
   do {
      res = Atomic_Read32(var);
   } while (res != Atomic_ReadIfEqualWrite32(var, res, res & val));
#endif

return res;
}

#if defined VM_64BIT
/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadOr64 --
 *
 *      Atomic read (returned), bitwise OR with a value, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint64
Atomic_ReadOr64(Atomic_uint64 *var, // IN/OUT
                uint64 val)         // IN
{
   uint64 res;

#if defined VM_ARM_64
   res = _VMATOM_X(ROP, 64, TRUE, &var->value, orr, val);
#else
   do {
      res = var->value;
   } while (res != Atomic_ReadIfEqualWrite64(var, res, res | val));
#endif

return res;
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadAnd64 --
 *
 *      Atomic read (returned), bitwise AND with a value, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint64
Atomic_ReadAnd64(Atomic_uint64 *var, // IN/OUT
                 uint64 val)         // IN
{
   uint64 res;

#if defined VM_ARM_64
   res = _VMATOM_X(ROP, 64, TRUE, &var->value, and, val);
#else
   do {
      res = var->value;
   } while (res != Atomic_ReadIfEqualWrite64(var, res, res & val));
#endif

return res;
}
#endif /* defined VM_64BIT */

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadAdd32 --
 *
 *      Atomic read (returned), add a value, write.
 *
 *      If you have to implement ReadAdd32() on an architecture other than
 *      x86 or x86-64, you might want to consider doing something similar to
 *      Atomic_ReadOr32().
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint32
Atomic_ReadAdd32(Atomic_uint32 *var, // IN/OUT
                 uint32 val)         // IN
{
#if defined __GNUC__
#ifdef VM_ARM_V7
   register volatile uint32 res;
   register volatile uint32 retVal;
   register volatile uint32 tmp;

dmb();

__asm__ __volatile__(
      "1: ldrex %[retVal], [%[var]] \n\t"
      "add %[tmp], %[val], %[retVal] \n\t"
      "strex %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [tmp] "=&r" (tmp), [res] "=&r" (res), [retVal] "=&r" (retVal)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();

return retVal;
#elif defined VM_ARM_64
   return _VMATOM_X(ROP, 32, TRUE, &var->value, add, val);
#else /* VM_X86_ANY */
   /* Checked against the Intel manual and GCC --walken */
   __asm__ __volatile__(
      "lock; xaddl %0, %1"
      : "=r" (val),
        "+m" (var->value)
      : "0" (val)
      : "cc", "memory"
   );
   return val;
#endif /* VM_X86_ANY */
#elif defined _MSC_VER
   return _InterlockedExchangeAdd((long *)&var->value, (long)val);
#else
#error No compiler defined for Atomic_ReadAdd32
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadInc32 --
 *
 *      Atomic read (returned), increment, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint32
Atomic_ReadInc32(Atomic_uint32 *var) // IN/OUT
{
   return Atomic_ReadAdd32(var, 1);
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadDec32 --
 *
 *      Atomic read (returned), decrement, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint32
Atomic_ReadDec32(Atomic_uint32 *var) // IN/OUT
{
   return Atomic_ReadAdd32(var, (uint32)-1);
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_CMPXCHG64 --
 *
 *      Compare exchange: Read variable, if equal to oldVal, write newVal
 *
 * Results:
 *      TRUE if equal, FALSE if not equal
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE Bool
Atomic_CMPXCHG64(Atomic_uint64 *var,   // IN/OUT
                 uint64 oldVal,        // IN
                 uint64 newVal)        // IN
{
#if defined __GNUC__
#if defined VM_ARM_ANY
   return Atomic_ReadIfEqualWrite64(var, oldVal, newVal) == oldVal;
#else /* VM_X86_ANY */

Bool equal;
   /* Checked against the Intel manual and GCC --walken */
#if defined __x86_64__
   uint64 dummy;
   __asm__ __volatile__(
      "lock; cmpxchgq %3, %0" "\n\t"
      "sete %1"
      : "+m" (*var),
        "=qm" (equal),
        "=a" (dummy)
      : "r" (newVal),
        "2" (oldVal)
      : "cc", "memory"
   );
#else /* 32-bit version for non-ARM */
   typedef struct {
      uint32 lowValue;
      uint32 highValue;
   } S_uint64;

int dummy1, dummy2;
#   if defined __PIC__
   /*
    * Rules for __asm__ statements in __PIC__ code
    * --------------------------------------------
    *
    * The compiler uses %ebx for __PIC__ code, so an __asm__ statement cannot
    * clobber %ebx. The __asm__ statement can temporarily modify %ebx, but _for
    * each parameter that is used while %ebx is temporarily modified_:
    *
    * 1) The constraint cannot be "m", because the memory location the compiler
    *    chooses could then be relative to %ebx.
    *
    * 2) The constraint cannot be a register class which contains %ebx (such as
    *    "r" or "q"), because the register the compiler chooses could then be
    *    %ebx. (This happens when compiling the Fusion UI with gcc 4.2.1, Apple
    *    build 5577.)
    *
    * 3) Using register classes even for other values is problematic, as gcc
    *    can decide e.g. %ecx == %edi == 0 (as compile-time constants) and
    *    ends up using one register for two things. Which breaks xchg's ability
    *    to temporarily put the PIC pointer somewhere else. PR772455
    *
    * For that reason alone, the __asm__ statement should keep the regions
    * where it temporarily modifies %ebx as small as possible, and should
    * prefer specific register assignments.
    */
   __asm__ __volatile__(
      "xchgl %%ebx, %6"      "\n\t"
      "lock; cmpxchg8b (%3)" "\n\t"
      "xchgl %%ebx, %6"      "\n\t"
      "sete %0"
      : "=qm" (equal),
        "=a" (dummy1),
        "=d" (dummy2)
      : /*
         * See the "Rules for __asm__ statements in __PIC__ code" above: %3
         * must use a register class which does not contain %ebx.
         * "a"/"c"/"d" are already used, so we are left with either "S" or "D".
         *
         * Note that this assembly uses ALL GP registers (with %esp reserved for
         * stack, %ebp reserved for frame, %ebx reserved for PIC).
         */
        "S" (var),
        "1" (((S_uint64 *)&oldVal)->lowValue),
        "2" (((S_uint64 *)&oldVal)->highValue),
        "D" (((S_uint64 *)&newVal)->lowValue),
        "c" (((S_uint64 *)&newVal)->highValue)
      : "cc", "memory"
   );
#   else
   __asm__ __volatile__(
      "lock; cmpxchg8b %0" "\n\t"
      "sete %1"
      : "+m" (*var),
        "=qm" (equal),
        "=a" (dummy1),
        "=d" (dummy2)
      : "2" (((S_uint64 *)&oldVal)->lowValue),
        "3" (((S_uint64 *)&oldVal)->highValue),
        "b" (((S_uint64 *)&newVal)->lowValue),
        "c" (((S_uint64 *)&newVal)->highValue)
      : "cc", "memory"
   );
#   endif
#endif
   return equal;
#endif //VM_ARM_V7
#elif defined _MSC_VER
   return (__int64)oldVal == _InterlockedCompareExchange64((__int64 *)&var->value,
                                                           (__int64)newVal,
                                                           (__int64)oldVal);
#else
#error No compiler defined for Atomic_CMPXCHG64
#endif // !GNUC
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_CMPXCHG32 --
 *
 *      Compare exchange: Read variable, if equal to oldVal, write newVal
 *
 * Results:
 *      TRUE if equal, FALSE if not equal
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE Bool
Atomic_CMPXCHG32(Atomic_uint32 *var,   // IN/OUT
                 uint32 oldVal,        // IN
                 uint32 newVal)        // IN
{
#if defined __GNUC__
#if defined VM_ARM_ANY
   return Atomic_ReadIfEqualWrite32(var, oldVal, newVal) == oldVal;
#else /* VM_X86_ANY */
   Bool equal;
   uint32 dummy;

__asm__ __volatile__(
      "lock; cmpxchgl %3, %0" "\n\t"
      "sete %1"
      : "+m" (*var),
        "=qm" (equal),
        "=a" (dummy)
      : "r" (newVal),
        "2" (oldVal)
      : "cc", "memory"
   );
   return equal;
#endif /* VM_X86_ANY */
#else // defined __GNUC__
   return Atomic_ReadIfEqualWrite32(var, oldVal, newVal) == oldVal;
#endif // !defined __GNUC__
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Read64 --
 *
 *      Read and return.
 *
 * Results:
 *      The value of the atomic variable.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint64
Atomic_Read64(Atomic_uint64 const *var) // IN
{
#if defined __GNUC__
   uint64 value;
#endif

#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE
   ASSERT((uintptr_t)var % 8 == 0);
#endif

#if defined __GNUC__ && defined __x86_64__
   /*
    * Use asm to ensure we emit a single load.
    */
   __asm__ __volatile__(
      "movq %1, %0"
      : "=r" (value)
      : "m" (var->value)
   );
#elif defined __GNUC__ && defined __i386__
   /*
    * Since cmpxchg8b will replace the contents of EDX:EAX with the
    * value in memory if there is no match, we need only execute the
    * instruction once in order to atomically read 64 bits from
    * memory.  The only constraint is that ECX:EBX must have the same
    * value as EDX:EAX so that if the comparison succeeds.  We
    * intentionally don't tell gcc that we are using ebx and ecx as we
    * don't modify them and do not care what value they store.
    */
   __asm__ __volatile__(
      "mov %%ebx, %%eax"   "\n\t"
      "mov %%ecx, %%edx"   "\n\t"
      "lock; cmpxchg8b %1"
      : "=&A" (value)
      : "m" (*var)
      : "cc"
   );
#elif defined _MSC_VER && defined __x86_64__
   /*
    * Microsoft docs guarantee "Simple reads and writes to properly
    * aligned 64-bit variables are atomic on 64-bit Windows."
    * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx
    *
    * XXX Verify that value is properly aligned. Bug 61315.
    */
   return var->value;
#elif defined _MSC_VER && defined VM_ARM_32
   return _InterlockedAdd64((__int64 *)&var->value, 0);
#elif defined _MSC_VER && defined __i386__
#   pragma warning(push)
#   pragma warning(disable : 4035)      // disable no-return warning
   {
      __asm mov ecx, var
      __asm mov edx, ecx
      __asm mov eax, ebx
      __asm lock cmpxchg8b [ecx]
      // edx:eax is the return value; this is documented to work. --mann
   }
#   pragma warning(pop)
#elif defined __GNUC__ && defined VM_ARM_V7
   __asm__ __volatile__(
      "ldrexd %[value], %H[value], [%[var]] \n\t"
      : [value] "=&r" (value)
      : [var] "r" (&var->value)
   );
#elif defined VM_ARM_64
   value = _VMATOM_X(R, 64, &var->value);
#endif

#if defined __GNUC__
   return value;
#endif
}

/*
 *----------------------------------------------------------------------
 *
 * Atomic_ReadUnaligned64 --
 *
 *      Atomically read a 64 bit integer, possibly misaligned.
 *      This function can be *very* expensive, costing over 50 kcycles
 *      on Nehalem.
 *
 *      Note that "var" needs to be writable, even though it will not
 *      be modified.
 *
 * Results:
 *      The value of the atomic variable.
 *
 * Side effects:
 *      None
 *
 *----------------------------------------------------------------------
 */

#if defined VM_64BIT
static INLINE uint64
Atomic_ReadUnaligned64(Atomic_uint64 const *var)  // IN:
{
   return Atomic_ReadIfEqualWrite64((Atomic_uint64*)var, 0, 0);
}
#endif

/*
 *----------------------------------------------------------------------
 *
 * Atomic_ReadAdd64 --
 *
 *      Atomically adds a 64-bit integer to another
 *
 * Results:
 *      Returns the old value just prior to the addition
 *
 * Side effects:
 *      None
 *
 *----------------------------------------------------------------------
 */

static INLINE uint64
Atomic_ReadAdd64(Atomic_uint64 *var, // IN/OUT
                 uint64 val)         // IN
{
#if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, add, val);
#elif defined __x86_64__

#if defined __GNUC__
   __asm__ __volatile__(
      "lock; xaddq %0, %1"
      : "=r" (val),
        "+m" (var->value)
      : "0" (val)
      : "cc", "memory"
   );
   return val;
#elif defined _MSC_VER
   return _InterlockedExchangeAdd64((__int64 *)&var->value, (__int64)val);
#else
#error No compiler defined for Atomic_ReadAdd64
#endif

#else
   uint64 oldVal;
   uint64 newVal;

do {
      oldVal = var->value;
      newVal = oldVal + val;
   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));

return oldVal;
#endif
}

/*
 *----------------------------------------------------------------------
 *
 * Atomic_ReadSub64 --
 *
 *      Atomically subtracts a 64-bit integer to another
 *
 * Results:
 *      Returns the old value just prior to the subtraction
 *
 * Side effects:
 *      None
 *
 *----------------------------------------------------------------------
 */

static INLINE uint64
Atomic_ReadSub64(Atomic_uint64 *var, // IN/OUT
                 uint64 val)         // IN
{
#if defined VM_ARM_64
   return _VMATOM_X(ROP, 64, TRUE, &var->value, sub, val);
#else
   return Atomic_ReadAdd64(var, -(int64)val);
#endif
}

/*
 *----------------------------------------------------------------------
 *
 * Atomic_ReadInc64 --
 *
 *      Atomically increments a 64-bit integer
 *
 * Results:
 *      Returns the old value just prior to incrementing
 *
 * Side effects:
 *      None
 *
 *----------------------------------------------------------------------
 */

static INLINE uint64
Atomic_ReadInc64(Atomic_uint64 *var) // IN/OUT
{
   return Atomic_ReadAdd64(var, 1);
}

/*
 *----------------------------------------------------------------------
 *
 * Atomic_ReadDec64 --
 *
 *      Atomically decrements a 64-bit integer
 *
 * Results:
 *      Returns the old value just prior to decrementing
 *
 * Side effects:
 *      None
 *
 *----------------------------------------------------------------------
 */

static INLINE uint64
Atomic_ReadDec64(Atomic_uint64 *var) // IN/OUT
{
   return Atomic_ReadAdd64(var, (uint64)CONST64(-1));
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Add64 --
 *
 *      Atomic read, add a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Add64(Atomic_uint64 *var, // IN/OUT
             uint64 val)         // IN
{
#if !defined VM_64BIT
   Atomic_ReadAdd64(var, val); /* Return value is unused. */
#elif defined __GNUC__
#if defined VM_ARM_64
   _VMATOM_X(OP, 64, TRUE, &var->value, add, val);
#else /* defined VM_X86_64 */
   /* Checked against the AMD manual and GCC --hpreg */
   __asm__ __volatile__(
      "lock; addq %1, %0"
      : "+m" (var->value)
      : "re" (val)
      : "cc", "memory"
   );
#endif
#elif defined _MSC_VER
   _InterlockedExchangeAdd64((__int64 *)&var->value, (__int64)val);
#else
#error No compiler defined for Atomic_Add64
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Sub64 --
 *
 *      Atomic read, subtract a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Sub64(Atomic_uint64 *var, // IN/OUT
             uint64 val)         // IN
{
#if !defined VM_64BIT
   Atomic_ReadSub64(var, val); /* Return value is unused. */
#elif defined __GNUC__
#if defined VM_ARM_64
   _VMATOM_X(OP, 64, TRUE, &var->value, sub, val);
#else /* VM_X86_64 */
   /* Checked against the AMD manual and GCC --hpreg */
   __asm__ __volatile__(
      "lock; subq %1, %0"
      : "+m" (var->value)
      : "re" (val)
      : "cc", "memory"
   );
#endif
#elif defined _MSC_VER
   _InterlockedExchangeAdd64((__int64 *)&var->value, (__int64)-val);
#else
#error No compiler defined for Atomic_Sub64
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Inc64 --
 *
 *      Atomic read, increment, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Inc64(Atomic_uint64 *var) // IN/OUT
{
#if defined VM_ARM_64
   Atomic_Add64(var, 1);
#elif !defined __x86_64__
   Atomic_ReadInc64(var);  /* Return value is unused. */
#elif defined __GNUC__
   /* Checked against the AMD manual and GCC --hpreg */
   __asm__ __volatile__(
      "lock; incq %0"
      : "+m" (var->value)
      :
      : "cc", "memory"
   );
#elif defined _MSC_VER
   _InterlockedIncrement64((__int64 *)&var->value);
#else
#error No compiler defined for Atomic_Inc64
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Dec64 --
 *
 *      Atomic read, decrement, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Dec64(Atomic_uint64 *var) // IN/OUT
{
#if defined VM_ARM_64
   Atomic_Sub64(var, 1);
#elif !defined __x86_64__
   Atomic_ReadDec64(var);  /* Return value is unused. */
#elif defined __GNUC__
   /* Checked against the AMD manual and GCC --hpreg */
   __asm__ __volatile__(
      "lock; decq %0"
      : "+m" (var->value)
      :
      : "cc", "memory"
   );
#elif defined _MSC_VER
   _InterlockedDecrement64((__int64 *)&var->value);
#else
#error No compiler defined for Atomic_Dec64
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadWrite64 --
 *
 *      Read followed by write
 *
 * Results:
 *      The value of the atomic variable before the write.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint64
Atomic_ReadWrite64(Atomic_uint64 *var, // IN/OUT
                   uint64 val)         // IN
{
#if defined __x86_64__
#if defined __GNUC__
   /* Checked against the AMD manual and GCC --hpreg */
   __asm__ __volatile__(
      "xchgq %0, %1"
      : "=r" (val),
        "+m" (var->value)
      : "0" (val)
      : "memory"
   );
   return val;
#elif defined _MSC_VER
   return _InterlockedExchange64((__int64 *)&var->value, (__int64)val);
#else
#error No compiler defined for Atomic_ReadWrite64
#endif
#elif defined VM_ARM_64
   return _VMATOM_X(RW, 64, TRUE, &var->value, val);
#else
   uint64 oldVal;

do {
      oldVal = var->value;
   } while (!Atomic_CMPXCHG64(var, oldVal, val));

return oldVal;
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Write64 --
 *
 *      Write
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Write64(Atomic_uint64 *var, // OUT
               uint64 val)         // IN
{
#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE
   ASSERT((uintptr_t)var % 8 == 0);
#endif

#if defined __x86_64__
#if defined __GNUC__
   /*
    * There is no move instruction for 64-bit immediate to memory, so unless
    * the immediate value fits in 32-bit (i.e. can be sign-extended), GCC
    * breaks the assignment into two movl instructions.  The code below forces
    * GCC to load the immediate value into a register first.
    */

__asm__ __volatile__(
      "movq %1, %0"
      : "=m" (var->value)
      : "r" (val)
   );
#elif defined _MSC_VER
   /*
    * Microsoft docs guarantee "Simple reads and writes to properly aligned
    * 64-bit variables are atomic on 64-bit Windows."
    * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx
    *
    * XXX Verify that value is properly aligned. Bug 61315.
    */

var->value = val;
#else
#error No compiler defined for Atomic_Write64
#endif
#elif defined VM_ARM_64
   _VMATOM_X(W, 64, &var->value, val);
#else
   (void)Atomic_ReadWrite64(var, val);
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Or64 --
 *
 *      Atomic read, bitwise OR with a 64-bit value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Or64(Atomic_uint64 *var, // IN/OUT
            uint64 val)         // IN
{
#if defined __x86_64__
#if defined __GNUC__
   /* Checked against the AMD manual and GCC --hpreg */
   __asm__ __volatile__(
      "lock; orq %1, %0"
      : "+m" (var->value)
      : "re" (val)
      : "cc", "memory"
   );
#elif defined _MSC_VER
   _InterlockedOr64((__int64 *)&var->value, (__int64)val);
#else
#error No compiler defined for Atomic_Or64
#endif
#elif defined VM_ARM_64
   _VMATOM_X(OP, 64, TRUE, &var->value, orr, val);
#else // __x86_64__
   uint64 oldVal;
   uint64 newVal;
   do {
      oldVal = var->value;
      newVal = oldVal | val;
   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_And64 --
 *
 *      Atomic read, bitwise AND with a 64-bit value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_And64(Atomic_uint64 *var, // IN/OUT
             uint64 val)         // IN
{
#if defined __x86_64__
#if defined __GNUC__
   /* Checked against the AMD manual and GCC --hpreg */
   __asm__ __volatile__(
      "lock; andq %1, %0"
      : "+m" (var->value)
      : "re" (val)
      : "cc", "memory"
   );
#elif defined _MSC_VER
   _InterlockedAnd64((__int64 *)&var->value, (__int64)val);
#else
#error No compiler defined for Atomic_And64
#endif
#elif defined VM_ARM_64
   _VMATOM_X(OP, 64, TRUE, &var->value, and, val);
#else // __x86_64__
   uint64 oldVal;
   uint64 newVal;
   do {
      oldVal = var->value;
      newVal = oldVal & val;
   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_SetBit64 --
 *
 *      Atomically set the bit 'bit' in var.  Bit must be between 0 and 63.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_SetBit64(Atomic_uint64 *var, // IN/OUT
                unsigned bit)       // IN
{
#if defined __x86_64__ && defined __GNUC__
   ASSERT(bit <= 63);
   __asm__ __volatile__(
      "lock; btsq %1, %0"
      : "+m" (var->value)
      : "ri" ((uint64)bit)
      : "cc", "memory"
   );
#else
   uint64 oldVal;
   uint64 newVal;
   ASSERT(bit <= 63);
   do {
      oldVal = var->value;
      newVal = oldVal | (CONST64U(1) << bit);
   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ClearBit64 --
 *
 *      Atomically clear the bit 'bit' in var.  Bit must be between 0 and 63.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_ClearBit64(Atomic_uint64 *var, // IN/OUT
                  unsigned bit)       // IN
{
#if defined __x86_64__ && defined __GNUC__
   ASSERT(bit <= 63);
   __asm__ __volatile__(
      "lock; btrq %1, %0"
      : "+m" (var->value)
      : "ri" ((uint64)bit)
      : "cc", "memory"
   );
#else
   uint64 oldVal;
   uint64 newVal;
   ASSERT(bit <= 63);
   do {
      oldVal = var->value;
      newVal = oldVal & ~(CONST64U(1) << bit);
   } while (!Atomic_CMPXCHG64(var, oldVal, newVal));
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_TestBit64 --
 *
 *      Read the bit 'bit' in var.  Bit must be between 0 and 63.
 *
 * Results:
 *      TRUE if the tested bit was set; else FALSE.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE Bool
Atomic_TestBit64(Atomic_uint64 *var, // IN
                 unsigned bit)       // IN
{
   Bool out;
   ASSERT(bit <= 63);
#if defined __x86_64__ && defined __GNUC__
   __asm__ __volatile__(
      "btq %2, %1; setc %0"
      : "=rm"(out)
      : "m" (var->value),
        "rJ" ((uint64)bit)
      : "cc"
   );
#else
   out = (var->value & (CONST64U(1) << bit)) != 0;
#endif
   return out;
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_TestSetBit64 --
 *
 *      Atomically test and set the bit 'bit' in var.
 *      Bit must be between 0 and 63.
 *
 * Results:
 *      TRUE if the tested bit was set; else FALSE.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE Bool
Atomic_TestSetBit64(Atomic_uint64 *var, // IN/OUT
                    unsigned bit)       // IN
{
#if defined __x86_64__ && defined __GNUC__
   Bool out;
   ASSERT(bit <= 63);
   __asm__ __volatile__(
      "lock; btsq %2, %1; setc %0"
      : "=rm" (out), "+m" (var->value)
      : "rJ" ((uint64)bit)
      : "cc", "memory"
   );
   return out;
#else
   uint64 oldVal;
   uint64 mask;
   ASSERT(bit <= 63);
   mask = CONST64U(1) << bit;
   do {
      oldVal = var->value;
   } while (!Atomic_CMPXCHG64(var, oldVal, oldVal | mask));
   return (oldVal & mask) != 0;
#endif
}

#if defined __GNUC__
/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Read16 --
 *
 *      Read and return.
 *
 * Results:
 *      The value of the atomic variable.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint16
Atomic_Read16(Atomic_uint16 const *var) // IN
{
   uint16 value;

#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE
   ASSERT((uintptr_t)var % 2 == 0);
#endif

#if defined __GNUC__
#if defined __x86_64__ || defined __i386__

__asm__ __volatile__(
      "movw %1, %0"
      : "=r" (value)
      : "m" (var->value)
   );
#elif defined VM_ARM_V7
   NOT_TESTED();

__asm__ __volatile__(
      "ldrh %0, [%1]"
      : "=r" (value)
      : "r" (&var->value)
   );
#elif defined VM_ARM_64
   value = _VMATOM_X(R, 16, &var->value);
#else
#error No 16-bits atomics.
#endif
#endif

return value;
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadWrite16 --
 *
 *      Read followed by write
 *
 * Results:
 *      The value of the atomic variable before the write.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint16
Atomic_ReadWrite16(Atomic_uint16 *var,  // IN/OUT:
                   uint16 val)          // IN:
{
#if defined __GNUC__
#if defined __x86_64__ || defined __i386__
   __asm__ __volatile__(
      "xchgw %0, %1"
      : "=r" (val),
        "+m" (var->value)
      : "0" (val)
      : "memory"
   );
   return val;
#elif defined VM_ARM_V7
   register volatile uint16 retVal;
   register volatile uint16 res;

NOT_TESTED();

dmb();

__asm__ __volatile__(
   "1: ldrexh %[retVal], [%[var]] \n\t"
      "strexh %[res], %[val], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [retVal] "=&r" (retVal), [res] "=&r" (res)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();

return retVal;
#elif defined VM_ARM_64
   return _VMATOM_X(RW, 16, TRUE, &var->value, val);
#else
#error No 16-bits atomics.
#endif
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Write16 --
 *
 *      Write
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Write16(Atomic_uint16 *var,  // OUT:
               uint16 val)          // IN:
{
#if defined VMM || defined VM_ARM_64 || defined VMKERNEL || defined VMKERNEL_MODULE
   ASSERT((uintptr_t)var % 2 == 0);
#endif

#if defined __GNUC__
#if defined __x86_64__ || defined __i386__
   __asm__ __volatile__(
      "movw %1, %0"
      : "=m" (var->value)
      : "r" (val)
   );
#elif defined VM_ARM_64
   _VMATOM_X(W, 16, &var->value, val);
#elif defined VM_ARM_32
   /*
    * Best left this way due to the intricacies of exclusive load/store
    * operations on legacy (32-bit) ARM.
    */
   Atomic_ReadWrite16(var, val);
#else
#error No 16-bits atomics.
#endif
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadIfEqualWrite16 --
 *
 *      Compare exchange: Read variable, if equal to oldVal, write newVal
 *
 * Results:
 *      The value of the atomic variable before the write.
 *
 * Side effects:
 *      The variable may be modified.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint16
Atomic_ReadIfEqualWrite16(Atomic_uint16 *var,   // IN/OUT
                          uint16 oldVal,        // IN
                          uint16 newVal)        // IN
{
#if defined __GNUC__
#if defined __x86_64__ || defined __i386__
   uint16 val;

__asm__ __volatile__(
      "lock; cmpxchgw %2, %1"
      : "=a" (val),
        "+m" (var->value)
      : "r" (newVal),
        "0" (oldVal)
      : "cc", "memory"
   );
   return val;
#elif defined VM_ARM_V7
   register uint16 retVal;
   register uint16 res;

NOT_TESTED();

dmb();

__asm__ __volatile__(
   "1: ldrexh %[retVal], [%[var]] \n\t"
      "mov %[res], #0 \n\t"
      "teq %[retVal], %[oldVal] \n\t"
      "strexheq %[res], %[newVal], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [retVal] "=&r" (retVal), [res] "=&r" (res)
      : [var] "r" (&var->value), [oldVal] "r" (oldVal), [newVal] "r" (newVal)
      : "cc"
   );

dmb();

return retVal;
#elif defined VM_ARM_64
   return _VMATOM_X(RIFEQW, 16, TRUE, &var->value, oldVal, newVal);
#else
#error No 16-bits atomics.
#endif
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_And16 --
 *
 *      Atomic read, bitwise AND with a 16-bit value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_And16(Atomic_uint16 *var, // IN/OUT
             uint16 val)         // IN
{
#if defined __GNUC__
#if defined __x86_64__ || defined __i386__
   __asm__ __volatile__(
      "lock; andw %1, %0"
      : "+m" (var->value)
      : "re" (val)
      : "cc", "memory"
   );
#elif defined VM_ARM_V7
   register volatile uint16 res;
   register volatile uint16 tmp;

NOT_TESTED();

dmb();

__asm__ __volatile__(
   "1: ldrexh %[tmp], [%[var]] \n\t"
      "and %[tmp], %[tmp], %[val] \n\t"
      "strexh %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [res] "=&r" (res), [tmp] "=&r" (tmp)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();
#elif defined VM_ARM_64
   _VMATOM_X(OP, 16, TRUE, &var->value, and, val);
#else
#error No 16-bits atomics.
#endif
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Or16 --
 *
 *      Atomic read, bitwise OR with a 16-bit value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Or16(Atomic_uint16 *var, // IN/OUT
            uint16 val)         // IN
{
#if defined __GNUC__
#if defined __x86_64__ || defined __i386__
   __asm__ __volatile__(
      "lock; orw %1, %0"
      : "+m" (var->value)
      : "re" (val)
      : "cc", "memory"
   );
#elif defined VM_ARM_V7
   register volatile uint16 res;
   register volatile uint16 tmp;

NOT_TESTED();

dmb();

__asm__ __volatile__(
   "1: ldrexh %[tmp], [%[var]] \n\t"
      "orr %[tmp], %[tmp], %[val] \n\t"
      "strexh %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [res] "=&r" (res), [tmp] "=&r" (tmp)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();
#elif defined VM_ARM_64
   _VMATOM_X(OP, 16, TRUE, &var->value, orr, val);
#else
#error No 16-bits atomics.
#endif
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Xor16 --
 *
 *      Atomic read, bitwise XOR with a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Xor16(Atomic_uint16 *var, // IN/OUT
             uint16 val)         // IN
{
#if defined __GNUC__
#if defined __x86_64__ || defined __i386__
   __asm__ __volatile__(
      "lock; xorw %1, %0"
      : "+m" (var->value)
      : "re" (val)
      : "cc", "memory"
   );
#elif defined VM_ARM_V7
   register volatile uint16 res;
   register volatile uint16 tmp;

NOT_TESTED();

dmb();

__asm__ __volatile__(
   "1: ldrexh %[tmp], [%[var]] \n\t"
      "eor %[tmp], %[tmp], %[val] \n\t"
      "strexh %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [res] "=&r" (res), [tmp] "=&r" (tmp)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();
#elif defined VM_ARM_64
   _VMATOM_X(OP, 16, TRUE, &var->value, eor, val);
#else
#error No 16-bits atomics.
#endif
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Add16 --
 *
 *      Atomic read, add a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Add16(Atomic_uint16 *var, // IN/OUT
             uint16 val)         // IN
{
#if defined __GNUC__
#if defined __x86_64__ || defined __i386__
   __asm__ __volatile__(
      "lock; addw %1, %0"
      : "+m" (var->value)
      : "re" (val)
      : "cc", "memory"
   );
#elif defined VM_ARM_V7
   register volatile uint16 res;
   register volatile uint16 tmp;

NOT_TESTED();

dmb();

__asm__ __volatile__(
   "1: ldrexh %[tmp], [%[var]] \n\t"
      "add %[tmp], %[tmp], %[val] \n\t"
      "strexh %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [res] "=&r" (res), [tmp] "=&r" (tmp)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();
#elif defined VM_ARM_64
   _VMATOM_X(OP, 16, TRUE, &var->value, add, val);
#else
#error No 16-bits atomics.
#endif
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Sub16 --
 *
 *      Atomic read, subtract a value, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Sub16(Atomic_uint16 *var, // IN/OUT
             uint16 val)         // IN
{
#if defined __GNUC__
#if defined __x86_64__ || defined __i386__
   __asm__ __volatile__(
      "lock; subw %1, %0"
      : "+m" (var->value)
      : "re" (val)
      : "cc", "memory"
   );
#elif defined VM_ARM_V7
   register volatile uint16 res;
   register volatile uint16 tmp;

NOT_TESTED();

dmb();

__asm__ __volatile__(
      "1: ldrexh %[tmp], [%[var]] \n\t"
      "sub %[tmp], %[tmp], %[val] \n\t"
      "strexh %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [res] "=&r" (res), [tmp] "=&r" (tmp)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();
#elif defined VM_ARM_64
   _VMATOM_X(OP, 16, TRUE, &var->value, sub, val);
#else
#error No 16-bits atomics.
#endif
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Inc16 --
 *
 *      Atomic read, increment, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Inc16(Atomic_uint16 *var) // IN/OUT
{
#if defined __GNUC__
#if defined __x86_64__ || defined __i386__
   __asm__ __volatile__(
      "lock; incw %0"
      : "+m" (var->value)
      :
      : "cc", "memory"
   );
#elif defined VM_ARM_ANY
   Atomic_Add16(var, 1);
#else
#error No 16-bits atomics.
#endif
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_Dec16 --
 *
 *      Atomic read, decrement, write.
 *
 * Results:
 *      None
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
Atomic_Dec16(Atomic_uint16 *var) // IN/OUT
{
#if defined __GNUC__
#if defined __x86_64__ || defined __i386__
   __asm__ __volatile__(
      "lock; decw %0"
      : "+m" (var->value)
      :
      : "cc", "memory"
   );
#elif defined VM_ARM_ANY
   Atomic_Sub16(var, 1);
#else
#error No 16-bits atomics.
#endif
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_ReadOr16 --
 *
 *      Atomic read (returned), bitwise OR with a value, write.
 *
 * Results:
 *      The value of the variable before the operation.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint16
Atomic_ReadOr16(Atomic_uint16 *var, // IN/OUT
                uint16 val)         // IN
{
   uint16 res;

#if defined VM_ARM_64
   res = _VMATOM_X(ROP, 16, TRUE, &var->value, orr, val);
#else
   do {
      res = var->value;
   } while (res != Atomic_ReadIfEqualWrite16(var, res, res | val));
#endif

return res;
}

/*
 *----------------------------------------------------------------------
 *
 * Atomic_ReadAdd16 --
 *
 *      Atomically adds a 16-bit integer to another
 *
 * Results:
 *      Returns the old value just prior to the addition
 *
 * Side effects:
 *      None
 *
 *----------------------------------------------------------------------
 */

static INLINE uint16
Atomic_ReadAdd16(Atomic_uint16 *var,  // IN/OUT
                 uint16 val)          // IN:
{
#if defined __GNUC__
#if defined __x86_64__ || defined __i386__
   __asm__ __volatile__(
      "lock; xaddw %0, %1"
      : "=r" (val),
        "+m" (var->value)
      : "0" (val)
      : "cc", "memory"
   );
   return val;
#elif defined VM_ARM_V7
   register volatile uint16 res;
   register volatile uint16 retVal;
   register volatile uint16 tmp;

NOT_TESTED();

dmb();

__asm__ __volatile__(
      "1: ldrexh %[retVal], [%[var]] \n\t"
      "add %[tmp], %[val], %[retVal] \n\t"
      "strexh %[res], %[tmp], [%[var]] \n\t"
      "teq %[res], #0 \n\t"
      "bne 1b"
      : [tmp] "=&r" (tmp), [res] "=&r" (res), [retVal] "=&r" (retVal)
      : [var] "r" (&var->value), [val] "r" (val)
      : "cc"
   );

dmb();

return retVal;
#elif defined VM_ARM_64
   return _VMATOM_X(ROP, 16, TRUE, &var->value, add, val);
#else
#error No 16-bits atomics.
#endif
#endif
}

/*
 *----------------------------------------------------------------------
 *
 * Atomic_ReadInc16 --
 *
 *      Atomically increments a 64-bit integer
 *
 * Results:
 *      Returns the old value just prior to incrementing
 *
 * Side effects:
 *      None
 *
 *----------------------------------------------------------------------
 */

static INLINE uint16
Atomic_ReadInc16(Atomic_uint16 *var) // IN/OUT
{
   return Atomic_ReadAdd16(var, 1);
}

/*
 *----------------------------------------------------------------------
 *
 * Atomic_ReadDec16 --
 *
 *      Atomically decrements a 64-bit integer
 *
 * Results:
 *      Returns the old value just prior to decrementing
 *
 * Side effects:
 *      None
 *
 *----------------------------------------------------------------------
 */

static INLINE uint16
Atomic_ReadDec16(Atomic_uint16 *var) // IN/OUT
{
   return Atomic_ReadAdd16(var, -1);
}
#endif

/*
 * Template code for the Atomic_<name> type and its operators.
 *
 * The cast argument is an intermediate type cast to make some
 * compilers stop complaining about casting uint32 <-> void *,
 * even though we only do it in the 32-bit case so they are always
 * the same size.  So for val of type uint32, instead of
 * (void *)val, we have (void *)(uintptr_t)val.
 * The specific problem case is the Windows ddk compiler
 * (as used by the SVGA driver).  -- edward
 *
 * NOTE: See the comment in vm_assert.h for why we need UNUSED_TYPE in
 * AtomicAssertOnCompile(), and why we need to be very careful doing so.
 */

#define MAKE_ATOMIC_TYPE(name, size, in, out, cast)                           \
   typedef Atomic_uint ## size Atomic_ ## name;                               \
                                                                              \
                                                                              \
   static INLINE void                                                         \
   AtomicAssertOnCompile ## name(void)                                        \
   {                                                                          \
      enum { AssertOnCompileMisused =    8 * sizeof (in) == size              \
                                      && 8 * sizeof (out) == size             \
                                      && 8 * sizeof (cast) == size            \
                                         ? 1 : -1 };                          \
      UNUSED_TYPE(typedef char AssertOnCompileFailed[AssertOnCompileMisused]);\
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE out                                                          \
   Atomic_Read ## name(Atomic_ ## name const *var)                            \
   {                                                                          \
      return (out)(cast)Atomic_Read ## size(var);                             \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE void                                                         \
   Atomic_Write ## name(Atomic_ ## name *var,                                 \
                        in val)                                               \
   {                                                                          \
      Atomic_Write ## size(var, (uint ## size)(cast)val);                     \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE out                                                          \
   Atomic_ReadWrite ## name(Atomic_ ## name *var,                             \
                            in val)                                           \
   {                                                                          \
      return (out)(cast)Atomic_ReadWrite ## size(var,                         \
                (uint ## size)(cast)val);                                     \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE out                                                          \
   Atomic_ReadIfEqualWrite ## name(Atomic_ ## name *var,                      \
                                   in oldVal,                                 \
                                   in newVal)                                 \
   {                                                                          \
      return (out)(cast)Atomic_ReadIfEqualWrite ## size(var,                  \
                (uint ## size)(cast)oldVal, (uint ## size)(cast)newVal);      \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE void                                                         \
   Atomic_And ## name(Atomic_ ## name *var,                                   \
                      in val)                                                 \
   {                                                                          \
      Atomic_And ## size(var, (uint ## size)(cast)val);                       \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE void                                                         \
   Atomic_Or ## name(Atomic_ ## name *var,                                    \
                     in val)                                                  \
   {                                                                          \
      Atomic_Or ## size(var, (uint ## size)(cast)val);                        \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE void                                                         \
   Atomic_Xor ## name(Atomic_ ## name *var,                                   \
                      in val)                                                 \
   {                                                                          \
      Atomic_Xor ## size(var, (uint ## size)(cast)val);                       \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE void                                                         \
   Atomic_Add ## name(Atomic_ ## name *var,                                   \
                      in val)                                                 \
   {                                                                          \
      Atomic_Add ## size(var, (uint ## size)(cast)val);                       \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE void                                                         \
   Atomic_Sub ## name(Atomic_ ## name *var,                                   \
                      in val)                                                 \
   {                                                                          \
      Atomic_Sub ## size(var, (uint ## size)(cast)val);                       \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE void                                                         \
   Atomic_Inc ## name(Atomic_ ## name *var)                                   \
   {                                                                          \
      Atomic_Inc ## size(var);                                                \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE void                                                         \
   Atomic_Dec ## name(Atomic_ ## name *var)                                   \
   {                                                                          \
      Atomic_Dec ## size(var);                                                \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE out                                                          \
   Atomic_ReadOr ## name(Atomic_ ## name *var,                                \
                         in val)                                              \
   {                                                                          \
      return (out)(cast)Atomic_ReadOr ## size(var, (uint ## size)(cast)val);  \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE out                                                          \
   Atomic_ReadAdd ## name(Atomic_ ## name *var,                               \
                          in val)                                             \
   {                                                                          \
      return (out)(cast)Atomic_ReadAdd ## size(var, (uint ## size)(cast)val); \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE out                                                          \
   Atomic_ReadInc ## name(Atomic_ ## name *var)                               \
   {                                                                          \
      return (out)(cast)Atomic_ReadInc ## size(var);                          \
   }                                                                          \
                                                                              \
                                                                              \
   static INLINE out                                                          \
   Atomic_ReadDec ## name(Atomic_ ## name *var)                               \
   {                                                                          \
      return (out)(cast)Atomic_ReadDec ## size(var);                          \
   }

/*
 * Since we use a macro to generate these definitions, it is hard to look for
 * them. So DO NOT REMOVE THIS COMMENT and keep it up-to-date. --hpreg
 *
 * Atomic_Ptr
 * Atomic_ReadPtr --
 * Atomic_WritePtr --
 * Atomic_ReadWritePtr --
 * Atomic_ReadIfEqualWritePtr --
 * Atomic_AndPtr --
 * Atomic_OrPtr --
 * Atomic_XorPtr --
 * Atomic_AddPtr --
 * Atomic_SubPtr --
 * Atomic_IncPtr --
 * Atomic_DecPtr --
 * Atomic_ReadOrPtr --
 * Atomic_ReadAddPtr --
 * Atomic_ReadIncPtr --
 * Atomic_ReadDecPtr --
 *
 * Atomic_Int
 * Atomic_ReadInt --
 * Atomic_WriteInt --
 * Atomic_ReadWriteInt --
 * Atomic_ReadIfEqualWriteInt --
 * Atomic_AndInt --
 * Atomic_OrInt --
 * Atomic_XorInt --
 * Atomic_AddInt --
 * Atomic_SubInt --
 * Atomic_IncInt --
 * Atomic_DecInt --
 * Atomic_ReadOrInt --
 * Atomic_ReadAddInt --
 * Atomic_ReadIncInt --
 * Atomic_ReadDecInt --
 *
 * Atomic_Bool
 * Atomic_ReadBool --
 * Atomic_WriteBool --
 * Atomic_ReadWriteBool --
 * Atomic_ReadIfEqualWriteBool --
 * Atomic_AndBool --
 * Atomic_OrBool --
 * Atomic_XorBool --
 * Atomic_AddBool --
 * Atomic_SubBool --
 * Atomic_IncBool --
 * Atomic_DecBool --
 * Atomic_ReadOrBool --
 * Atomic_ReadAddBool --
 * Atomic_ReadIncBool --
 * Atomic_ReadDecBool --
 */
#if defined VM_64BIT
MAKE_ATOMIC_TYPE(Ptr, 64, void const *, void *, uintptr_t)
#else
MAKE_ATOMIC_TYPE(Ptr, 32, void const *, void *, uintptr_t)
#endif
MAKE_ATOMIC_TYPE(Int, 32, int, int, int)
MAKE_ATOMIC_TYPE(Bool, 8, Bool, Bool, Bool)

/*
 * Define arbitrary sized bit vector to be used by
 * Atomic_TestSetBitVector and Atomic_TestClearBitVector.
 */
#define ATOMIC_BITVECTOR(varName, capacity) \
      Atomic_uint8 varName[CEILING(capacity, 8)]

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_TestSetBitVector --
 *
 *      Atomically test and set the bit 'index' in bit vector var.
 *
 *      The index input value specifies which bit to modify and is 0-based.
 *
 * Results:
 *      Returns the value of the bit before modification.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE Bool
Atomic_TestSetBitVector(Atomic_uint8 *var, // IN/OUT
                        unsigned index)    // IN
{
#if defined __x86_64__ && defined __GNUC__
   Bool bit;
   __asm__ __volatile__(
      "lock; bts %2, %1;"
      "setc %0"
      : "=qQm" (bit), "+m" (var->value)
      : "rI" (index)
      : "cc", "memory"
   );
   return bit;
#else
   uint8 bit = 1 << index % 8;
   return (Atomic_ReadOr8(var + index / 8, bit) & bit) != 0;
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_TestClearBitVector --
 *
 *      Atomically test and clear the bit 'index' in bit vector var.
 *
 *      The index input value specifies which bit to modify and is 0-based.
 *
 * Results:
 *      Returns the value of the bit before modification.
 *
 * Side effects:
 *      None
 *
 *-----------------------------------------------------------------------------
 */

static INLINE Bool
Atomic_TestClearBitVector(Atomic_uint8 *var, // IN/OUT
                          unsigned index)    // IN
{
#if defined __x86_64__ && defined __GNUC__
   Bool bit;
   __asm__ __volatile__(
      "lock; btr %2, %1;"
      "setc %0"
      : "=qQm" (bit), "+m" (var->value)
      : "rI" (index)
      : "cc", "memory"
   );
   return bit;
#else
   uint8 bit = 1 << index % 8;
   return (Atomic_ReadAnd8(var + index / 8, ~bit) & bit) != 0;
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Atomic_TestBitVector --
 *
 *      Test the bit 'index' (zero-based) in bit vector var.
 *-----------------------------------------------------------------------------
 */

static INLINE Bool
Atomic_TestBitVector(const Atomic_uint8 *var, // IN
                     unsigned index)          // IN
{
   uint8 bit = 1 << index % 8;
   return (Atomic_Read8(var + index / 8) & bit) != 0;
}

#ifdef VM_ARM_64
#   include "vm_atomic_arm64_end.h"
#endif

#if defined __cplusplus
}  // extern "C"
#endif

#endif // ifndef _ATOMIC_H_
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vm_basic_types.h����������������������������������������������������������������0000444�0000000�0000000�00000070343�13432725350�016576� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 *
 * vm_basic_types.h --
 *
 *    basic data types.
 */

#ifndef _VM_BASIC_TYPES_H_
#define _VM_BASIC_TYPES_H_

#define INCLUDE_ALLOW_USERLEVEL

/*
 * Macros __i386__ and __ia64 are intrinsically defined by GCC
 */
#if defined _MSC_VER && defined _M_X64
#  define __x86_64__
#elif defined _MSC_VER && defined _M_IX86
#  define __i386__
#endif

/*
 * Setup a bunch of defines for instruction set architecture (ISA) related
 * properties.
 *
 * For compiler types/size:
 *
 * - VM_32BIT for a 32-bit ISA (with the same C compiler types/sizes as 32-bit
 *   x86/ARM).
 * - VM_64BIT for a 64-bit ISA (with the same C compiler types/sizes as 64-bit
 *   x86/ARM).
 *
 * For a given <arch> in {X86, ARM}:
 *
 * - VM_<arch>_32 for the 32-bit variant.
 * - VM_<arch>_64 for the 64-bit variant.
 * - VM_<arch>_ANY for any variant of <arch>.
 *
 * VM_X86_ANY is synonymous with the confusing and deprecated VM_I386 (which
 * should really be VM_X86_32).
 */

#ifdef __i386__
/*
 * VM_I386 is historically synonymous with VM_X86_ANY in bora, but misleading,
 * since it is confused with the __i386__ gcc but defined for both 32- and
 * 64-bit x86. We retain it here for legacy compatibility.
 */
#define VM_I386
#define VM_X86_32
#define VM_X86_ANY
#define VM_32BIT
#endif

#ifdef __x86_64__
#define VM_X86_64
#define vm_x86_64 1
#define VM_I386
#define VM_X86_ANY
#define VM_64BIT
#else
#define vm_x86_64 0
#endif

#ifdef __arm__
#define VM_ARM_32
#define VM_ARM_ANY
#define VM_32BIT
#endif

#ifdef __aarch64__
#define VM_ARM_64
#define vm_arm_64 1
#define VM_ARM_ANY
#define VM_64BIT
#else
#define vm_arm_64 0
#endif

#define vm_64bit (sizeof (void *) == 8)

#ifdef _MSC_VER

#pragma warning (3 :4505) // unreferenced local function
#pragma warning (disable :4018) // signed/unsigned mismatch
#pragma warning (disable :4761) // integral size mismatch in argument; conversion supplied
#pragma warning (disable :4305) // truncation from 'const int' to 'short'
#pragma warning (disable :4244) // conversion from 'unsigned short' to 'unsigned char'
#pragma warning (disable :4267) // truncation of 'size_t'
#pragma warning (disable :4146) // unary minus operator applied to unsigned type, result still unsigned
#pragma warning (disable :4142) // benign redefinition of type

#endif

#if defined(__cplusplus) && __cplusplus >= 201103L || \
    defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L || \
    defined(__APPLE__) || defined(HAVE_STDINT_H)
/*
 * We're using <stdint.h> instead of <cstdint> below because some C++ code
 * deliberately compiles without C++ include paths.
 */
#include <stdint.h>

typedef uint64_t    uint64;
typedef  int64_t     int64;
typedef uint32_t    uint32;
typedef  int32_t     int32;
typedef uint16_t    uint16;
typedef  int16_t     int16;
typedef  uint8_t     uint8;
typedef   int8_t      int8;

#else /* !HAVE_STDINT_H */

/* Pre-c99 or pre-c++11; use compiler extension to get 64-bit types */
#ifdef _MSC_VER

typedef unsigned __int64 uint64;
typedef signed __int64 int64;

#elif __GNUC__
#   if defined(VM_X86_64) || defined(VM_ARM_64)
typedef unsigned long uint64;
typedef long int64;
#   else
/*
 * Only strict c90 (without extensions) lacks a 'long long' type.
 * If this declaration fails ... use -std=c99 or -std=gnu90.
 */
typedef unsigned long long uint64;
typedef long long int64;
#   endif
#else
#   error - Need compiler define for int64/uint64
#endif /* _MSC_VER */

typedef unsigned int       uint32;
typedef unsigned short     uint16;
typedef unsigned char      uint8;

typedef int                int32;
typedef short              int16;
typedef signed char        int8;

#endif /* HAVE_STDINT_H */

/*
 * The _XTYPEDEF_BOOL guard prevents colliding with:
 * <X11/Xlib.h> #define Bool int
 * <X11/Xdefs.h> typedef int Bool;
 * If using this header AND X11 headers, be sure to #undef Bool and
 * be careful about the different size.
 */
#if !defined(_XTYPEDEF_BOOL)
#define _XTYPEDEF_BOOL
/*
 * C does not specify whether char is signed or unsigned, and
 * both gcc and msvc implement it as a non-signed, non-unsigned type.
 * Thus, (uint8_t *)&Bool and (int8_t *)&Bool are possible compile errors.
 * This is intentional.
 */
typedef char           Bool;
#endif

#ifndef FALSE
#define FALSE          0
#endif

#ifndef TRUE
#define TRUE           1
#endif

#define IS_BOOL(x)     (((x) & ~1) == 0)

/*
 * FreeBSD (for the tools build) unconditionally defines these in
 * sys/inttypes.h so don't redefine them if this file has already
 * been included. [greg]
 *
 * This applies to Solaris as well.
 */

/*
 * Before trying to do the includes based on OS defines, see if we can use
 * feature-based defines to get as much functionality as possible
 */

#ifdef HAVE_INTTYPES_H
#include <inttypes.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_INTTYPES_H
#include <sys/inttypes.h>
#endif
#ifdef HAVE_STDLIB_H
#include <stdlib.h>
#endif

#ifdef __FreeBSD__
#include <sys/param.h> /* For __FreeBSD_version */
#endif

#if !defined(USING_AUTOCONF)
#   if defined(__FreeBSD__) || defined(sun)
#      ifdef KLD_MODULE
#         include <sys/types.h>
#      else
#         if __FreeBSD_version >= 500043
#            if !defined(VMKERNEL)
#               include <inttypes.h>
#            endif
#            include <sys/types.h>
#         else
#            include <sys/inttypes.h>
#         endif
#      endif
#   elif defined __APPLE__
#      if KERNEL
#         include <sys/unistd.h>
#         include <sys/types.h> /* mostly for size_t */
#         include <stdint.h>
#      else
#         include <unistd.h>
#         include <inttypes.h>
#         include <stdlib.h>
#         include <stdint.h>
#      endif
#   elif defined __ANDROID__
#      include <stdint.h>
#   else
#      if !defined(__intptr_t_defined) && !defined(intptr_t)
#         ifdef VM_I386
#            define __intptr_t_defined
#            if defined(VM_X86_64)
typedef int64     intptr_t;
#            else
typedef int32     intptr_t;
#            endif
#         elif defined(VM_ARM_64)
#            define __intptr_t_defined
typedef int64     intptr_t;
#         elif defined(__arm__)
#            define __intptr_t_defined
typedef int32     intptr_t;
#         endif
#      endif

#      ifndef _STDINT_H
#         ifdef VM_I386
#            if defined(VM_X86_64)
typedef uint64    uintptr_t;
#            else
typedef uint32    uintptr_t;
#            endif
#         elif defined(VM_ARM_64)
typedef uint64    uintptr_t;
#         elif defined(__arm__)
typedef uint32    uintptr_t;
#         endif
#      endif
#   endif
#endif

#if defined(__GNUC__) && defined(__SIZEOF_INT128__)

typedef unsigned __int128 uint128;
typedef          __int128  int128;

#define MIN_INT128   ((int128)1 << 127)
#define MAX_INT128   (~MIN_INT128)
#define MIN_UINT128  ((uint128)0)
#define MAX_UINT128  (~MIN_UINT128)

#endif

/*
 * Time
 * XXX These should be cleaned up.  -- edward
 */

typedef int64 VmTimeType;          /* Time in microseconds */
typedef int64 VmTimeRealClock;     /* Real clock kept in microseconds */
typedef int64 VmTimeVirtualClock;  /* Virtual Clock kept in CPU cycles */

/*
 * Printf format specifiers for size_t and 64-bit number.
 * Use them like this:
 *    printf("%" FMT64 "d\n", big);
 * The spaces are important for C++11 compatibility.
 *
 * FMTH is for handles/fds.
 */

#ifdef _MSC_VER
   /* MSVC added C99-compatible formatting in vs2015. */
   #define FMT64      "I64"
   #ifdef VM_X86_64
      #define FMTSZ      "I64"
      #define FMTPD      "I64"
      #define FMTH       "I64"
   #else
      #define FMTSZ      "I"
      #define FMTPD      "I"
      #define FMTH       "I"
   #endif
#elif defined __APPLE__
   /* macOS hosts use the same formatters for 32- and 64-bit. */
   #define FMT64         "ll"
   #if KERNEL
      /* macOS osfmk/kern added 'z' length specifier in 10.13 */
      #define FMTSZ      "l"
   #else
      #define FMTSZ      "z"
   #endif
   #define FMTPD         "l"
   #define FMTH          ""
#elif defined __GNUC__
   /*
    * Every POSIX system we target has C99-compatible printf
    * (supports 'z' for size_t and 'll' for long long).
    */
   #define FMTH          ""
   #define FMTSZ         "z"
   #if defined(VM_X86_64) || defined(VM_ARM_64)
      #define FMT64      "l"
      #define FMTPD      "l"
   #else
      #define FMT64      "ll"
      #define FMTPD      ""
   #endif
#else
   #error - Need compiler define for FMT64 and FMTSZ
#endif

/*
 * Suffix for 64-bit constants.  Use it like this:
 *    CONST64(0x7fffffffffffffff) for signed or
 *    CONST64U(0x7fffffffffffffff) for unsigned.
 *
 * 2004.08.30(thutt):
 *   The vmcore/asm64/gen* programs are compiled as 32-bit
 *   applications, but must handle 64 bit constants.  If the
 *   64-bit-constant defining macros are already defined, the
 *   definition will not be overwritten.
 */

#if !defined(CONST64) || !defined(CONST64U)
#ifdef _MSC_VER
#define CONST64(c) c##I64
#define CONST64U(c) c##uI64
#elif defined __APPLE__
#define CONST64(c) c##LL
#define CONST64U(c) c##uLL
#elif __GNUC__
#if defined(VM_X86_64) || defined(VM_ARM_64)
#define CONST64(c) c##L
#define CONST64U(c) c##uL
#else
#define CONST64(c) c##LL
#define CONST64U(c) c##uLL
#endif
#else
#error - Need compiler define for CONST64
#endif
#endif

/*
 * Use CONST3264/CONST3264U if you want a constant to be
 * treated as a 32-bit number on 32-bit compiles and
 * a 64-bit number on 64-bit compiles. Useful in the case
 * of shifts, like (CONST3264U(1) << x), where x could be
 * more than 31 on a 64-bit compile.
 */

#if defined(VM_X86_64) || defined(VM_ARM_64)
    #define CONST3264(a) CONST64(a)
    #define CONST3264U(a) CONST64U(a)
#else
    #define CONST3264(a) (a)
    #define CONST3264U(a) (a)
#endif

#define MIN_INT8   ((int8)0x80)
#define MAX_INT8   ((int8)0x7f)

#define MIN_UINT8  ((uint8)0)
#define MAX_UINT8  ((uint8)0xff)

#define MIN_INT16  ((int16)0x8000)
#define MAX_INT16  ((int16)0x7fff)

#define MIN_UINT16 ((uint16)0)
#define MAX_UINT16 ((uint16)0xffff)

#define MIN_INT32  ((int32)0x80000000)
#define MAX_INT32  ((int32)0x7fffffff)

#define MIN_UINT32 ((uint32)0)
#define MAX_UINT32 ((uint32)0xffffffff)

#define MIN_INT64  (CONST64(0x8000000000000000))
#define MAX_INT64  (CONST64(0x7fffffffffffffff))

#define MIN_UINT64 (CONST64U(0))
#define MAX_UINT64 (CONST64U(0xffffffffffffffff))

typedef uint8 *TCA;  /* Pointer into TC (usually). */

/*
 * Type big enough to hold an integer between 0..100
 */
typedef uint8 Percent;
#define AsPercent(v)	((Percent)(v))

typedef uintptr_t VA;
typedef uintptr_t VPN;

typedef uint64    PA;
typedef uint32    PPN;

typedef uint64    TPA;
typedef uint32    TPPN;

typedef uint64    PhysMemOff;
typedef uint64    PhysMemSize;

typedef uint64    BA;
#ifdef VMKERNEL
typedef void     *BPN;
#else
typedef uint64    BPN;
#endif

#define UINT64_2_BPN(u) ((BPN)(u))
#define BPN_2_UINT64(b) ((uint64)(b))

typedef uint32    PageCnt;
typedef uint64    PgCnt64;
typedef uint32    PageNum;
typedef uint64    PgNum64;
typedef unsigned      MemHandle;
typedef unsigned int  IoHandle;
typedef int32     World_ID;

/* !! do not alter the definition of INVALID_WORLD_ID without ensuring
 * that the values defined in both bora/public/vm_basic_types.h and
 * lib/vprobe/vm_basic_types.h are the same.  Additionally, the definition
 * of VMK_INVALID_WORLD_ID in vmkapi_world.h also must be defined with
 * the same value
 */

#define INVALID_WORLD_ID ((World_ID)0)

typedef World_ID User_CartelID;
#define INVALID_CARTEL_ID INVALID_WORLD_ID

typedef User_CartelID User_SessionID;
#define INVALID_SESSION_ID INVALID_CARTEL_ID

typedef User_CartelID User_CartelGroupID;
#define INVALID_CARTELGROUP_ID INVALID_CARTEL_ID

typedef uint32 Worldlet_ID;
#define INVALID_WORLDLET_ID ((Worldlet_ID)-1)

typedef  int8    Reg8;
typedef  int16   Reg16;
typedef  int32   Reg32;
typedef  int64   Reg64;

typedef uint8   UReg8;
typedef uint16  UReg16;
typedef uint32  UReg32;
typedef uint64  UReg64;

#if defined(__GNUC__) && defined(__SIZEOF_INT128__)
typedef  int128  Reg128;
typedef uint128 UReg128;
#endif

#if defined(VMM) || defined(COREQUERY) || defined(EXTDECODER) ||  \
    defined (VMKERNEL) || defined (VMKBOOT)
typedef  Reg64  Reg;
typedef UReg64 UReg;
#endif
typedef uint64 MA;
typedef uint32 MPN32;

/*
 * This type should be used for variables that contain sector
 * position/quantity.
 */
typedef uint64 SectorType;

/*
 * Linear address
 */

typedef uintptr_t LA;
typedef uintptr_t LPN;
#define LA_2_LPN(_la)     ((_la) >> PAGE_SHIFT)
#define LPN_2_LA(_lpn)    ((_lpn) << PAGE_SHIFT)

#define LAST_LPN   ((((LA)  1) << (8 * sizeof(LA)   - PAGE_SHIFT)) - 1)
#define LAST_LPN32 ((((LA32)1) << (8 * sizeof(LA32) - PAGE_SHIFT)) - 1)
#define LAST_LPN64 ((((LA64)1) << (8 * sizeof(LA64) - PAGE_SHIFT)) - 1)

/* Valid bits in a LPN. */
#define LPN_MASK   LAST_LPN
#define LPN_MASK32 LAST_LPN32
#define LPN_MASK64 LAST_LPN64

/*
 * On 64 bit platform, address and page number types default
 * to 64 bit. When we need to represent a 32 bit address, we use
 * types defined below.
 *
 * On 32 bit platform, the following types are the same as the
 * default types.
 */
typedef uint32 VA32;
typedef uint32 VPN32;
typedef uint32 LA32;
typedef uint32 LPN32;
typedef uint32 PA32;
typedef uint32 PPN32;

/*
 * On 64 bit platform, the following types are the same as the
 * default types.
 */
typedef uint64 VA64;
typedef uint64 VPN64;
typedef uint64 LA64;
typedef uint64 LPN64;
typedef uint64 PA64;
typedef uint64 PPN64;
typedef uint64 TPPN64;
typedef uint64 MA64;
typedef uint64 MPN;

/*
 * Remove after PPN->PPN64 conversion is finished.
 */
#define PPN64_2_PPN(_ppn)     ((PPN)_ppn)

#define FMTPPN ""

/*
 * IO device DMA virtual address and page number (translated by IOMMU to
 * MA/MPN). IOPN can be in the inclusive range 0 -> MAX_IOPN.
 */
typedef uint64 IOA;
typedef uint64 IOPN;

/*
 * VA typedefs for user world apps.
 */
typedef VA32 UserVA32;
typedef VA64 UserVA64;
typedef UserVA64 UserVAConst; /* Userspace ptr to data that we may only read. */
typedef UserVA32 UserVA32Const; /* Userspace ptr to data that we may only read. */
typedef UserVA64 UserVA64Const; /* Used by 64-bit syscalls until conversion is finished. */
#ifdef VMKERNEL
typedef UserVA64 UserVA;
#else
typedef void * UserVA;
#endif

#define MAX_PPN_BITS      31
#define MAX_PPN           (((PPN)1 << MAX_PPN_BITS) - 1) /* Maximal observable PPN value. */
#define INVALID_PPN       ((PPN)0xffffffff)
#define INVALID_PPN32     ((PPN32)0xffffffff)
#define INVALID_PPN64     ((PPN64)0xffffffffffffffffull)
#define APIC_INVALID_PPN  ((PPN)0xfffffffe)

#define INVALID_BPN       ((BPN)0x000000ffffffffffull)

#define MPN38_MASK        ((1ull << 38) - 1)

#define RESERVED_MPN      ((MPN)0)
#define INVALID_MPN       ((MPN)MPN38_MASK)
#define MEMREF_MPN        ((MPN)MPN38_MASK - 1)
#define RELEASED_MPN      ((MPN)MPN38_MASK - 2)

/* account for special MPNs defined above */
#define MAX_MPN           ((MPN)MPN38_MASK - 3) /* 50 bits of address space */

#define INVALID_IOPN      ((IOPN)-1)
#define MAX_IOPN          (INVALID_IOPN - 1)

#define INVALID_LPN       ((LPN)-1)
#define INVALID_VPN       ((VPN)-1)
#define INVALID_LPN64     ((LPN64)-1)
#define INVALID_PAGENUM   ((PageNum)-1)

/*
 * Format modifier for printing VA, LA, and VPN.
 * Use them like this: Log("%#" FMTLA "x\n", laddr)
 */

#if defined(VMM) || defined(FROBOS64) || vm_x86_64 || vm_arm_64 || defined __APPLE__
#   define FMTLA "l"
#   define FMTVA "l"
#   define FMTVPN "l"
#else
#   define FMTLA ""
#   define FMTVA ""
#   define FMTVPN ""
#endif

#ifndef EXTERN
#define EXTERN        extern
#endif
#define CONST         const

#ifndef INLINE
#   ifdef _MSC_VER
       /*
        * On UWP(Universal Windows Platform),
        * Only X86 32bit support '__inline'
        */
#      if defined(VM_WIN_UWP) && !defined(VM_X86_32)
#            define INLINE
#      else
#            define INLINE        __inline
#      endif
#   else
#      define INLINE        inline
#   endif
#endif

/*
 * Annotation for data that may be exported into a DLL and used by other
 * apps that load that DLL and import the data.
 */
#if defined(_WIN32) && defined(VMX86_IMPORT_DLLDATA)
#  define VMX86_EXTERN_DATA       extern __declspec(dllimport)
#else // !_WIN32
#  define VMX86_EXTERN_DATA       extern
#endif

#ifdef _WIN32

/* under windows, __declspec(thread) is supported since VS 2003 */
#define __thread __declspec(thread)

#else

/*
 * under other platforms instead, __thread is supported by gcc since
 * version 3.3.1 and by clang since version 3.x
 */

#endif

/*
 * Due to the wonderful "registry redirection" feature introduced in
 * 64-bit Windows, if you access any key under HKLM\Software in 64-bit
 * code, you need to open/create/delete that key with
 * VMKEY_WOW64_32KEY if you want a consistent view with 32-bit code.
 */

#ifdef _WIN32
#ifdef _WIN64
#define VMW_KEY_WOW64_32KEY KEY_WOW64_32KEY
#else
#define VMW_KEY_WOW64_32KEY 0x0
#endif
#endif

/*
 * At present, we effectively require a compiler that is at least
 * gcc-3.3 (circa 2003).  Enforce this here, various things below
 * this line depend upon it.
 *
 * In practice, most things presently compile with gcc-4.1 or gcc-4.4.
 * The various linux kernel modules may use older (gcc-3.3) compilers.
 */
#if defined __GNUC__ && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 3))
#error "gcc version is too old to compile assembly, need gcc-3.3 or better"
#endif

/*
 * Similarly, we require a compiler that is at least vc80 (vs2005).
 * Enforce this here.
 */
#if defined _MSC_VER && _MSC_VER < 1400
#error "cl.exe version is too old, need vc80 or better"
#endif

/*
 * Consider the following reasons functions are inlined:
 *
 *  1) inlined for performance reasons
 *  2) inlined because it's a single-use function
 *
 * Functions which meet only condition 2 should be marked with this
 * inline macro; It is not critical to be inlined (but there is a
 * code-space & runtime savings by doing so), so when other callers
 * are added the inline-ness should be removed.
 */

#if defined __GNUC__
/*
 * Starting at version 3.3, gcc does not always inline functions marked
 * 'inline' (it depends on their size and other factors). To force gcc
 * to inline a function, one must use the __always_inline__ attribute.
 * This attribute should be used sparingly and with care.  It is usually
 * preferable to let gcc make its own inlining decisions
 */
#   define INLINE_ALWAYS INLINE __attribute__((__always_inline__))
#else
#   define INLINE_ALWAYS INLINE
#endif
#define INLINE_SINGLE_CALLER INLINE_ALWAYS

/*
 * Used when a hard guarantee of no inlining is needed. Very few
 * instances need this since the absence of INLINE is a good hint
 * that gcc will not do inlining.
 */

#if defined(__GNUC__)
#define ABSOLUTELY_NOINLINE __attribute__((__noinline__))
#elif defined(_MSC_VER)
#define ABSOLUTELY_NOINLINE __declspec(noinline)
#endif

/*
 * Used when a function has no effects except the return value and the
 * return value depends only on the parameters and/or global variables
 * Such a function can be subject to common subexpression elimination
 * and loop optimization just as an arithmetic operator would be.
 */

#if defined(__GNUC__) && (defined(VMM) || defined (VMKERNEL))
#define SIDE_EFFECT_FREE __attribute__((__pure__))
#else
#define SIDE_EFFECT_FREE
#endif

/*
 * Used when a function exmaines no input other than its arguments and
 * has no side effects other than its return value.  Stronger than
 * SIDE_EFFECT_FREE as the function is not allowed to read from global
 * memory.
 */

#if defined(__GNUC__) && (defined(VMM) || defined (VMKERNEL))
#define CONST_FUNCTION __attribute__((__const__))
#else
#define CONST_FUNCTION
#endif

/*
 * Attributes placed on function declarations to tell the compiler
 * that the function never returns.
 */

#ifdef _MSC_VER
#define NORETURN __declspec(noreturn)
#elif defined __GNUC__
#define NORETURN __attribute__((__noreturn__))
#else
#define NORETURN
#endif

/*
 * Static profiling hints for functions.
 *    A function can be either hot, cold, or neither.
 *    It is an error to specify both hot and cold for the same function.
 *    Note that there is no annotation for "neither."
 */

#if defined __GNUC__ && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
#define HOT __attribute__((hot))
#define COLD __attribute__((cold))
#else
#define HOT
#define COLD
#endif

/*
 * Branch prediction hints:
 *     LIKELY(exp)   - Expression exp is likely TRUE.
 *     UNLIKELY(exp) - Expression exp is likely FALSE.
 *   Usage example:
 *        if (LIKELY(excCode == EXC_NONE)) {
 *               or
 *        if (UNLIKELY(REAL_MODE(vc))) {
 *
 * We know how to predict branches on gcc3 and later (hopefully),
 * all others we don't so we do nothing.
 */

#if defined __GNUC__
/*
 * gcc3 uses __builtin_expect() to inform the compiler of an expected value.
 * We use this to inform the static branch predictor. The '!!' in LIKELY
 * will convert any !=0 to a 1.
 */
#define LIKELY(_exp)     __builtin_expect(!!(_exp), 1)
#define UNLIKELY(_exp)   __builtin_expect((_exp), 0)
#else
#define LIKELY(_exp)      (_exp)
#define UNLIKELY(_exp)    (_exp)
#endif

/*
 * GCC's argument checking for printf-like functions
 * This is conditional until we have replaced all `"%x", void *'
 * with `"0x%08x", (uint32) void *'. Note that %p prints different things
 * on different platforms.  Argument checking is enabled for the
 * vmkernel, which has already been cleansed.
 *
 * fmtPos is the position of the format string argument, beginning at 1
 * varPos is the position of the variable argument, beginning at 1
 */

#if defined(__GNUC__)
# define PRINTF_DECL(fmtPos, varPos) __attribute__((__format__(__printf__, fmtPos, varPos)))
#else
# define PRINTF_DECL(fmtPos, varPos)
#endif

#if defined(__GNUC__)
# define SCANF_DECL(fmtPos, varPos) __attribute__((__format__(__scanf__, fmtPos, varPos)))
#else
# define SCANF_DECL(fmtPos, varPos)
#endif

/*
 * UNUSED_PARAM should surround the parameter name and type declaration,
 * e.g. "int MyFunction(int var1, UNUSED_PARAM(int var2))"
 *
 */

#ifndef UNUSED_PARAM
# if defined(__GNUC__)
#  define UNUSED_PARAM(_parm) _parm  __attribute__((__unused__))
# elif defined _MSC_VER
#  define UNUSED_PARAM(_parm) __pragma(warning(suppress:4100)) _parm
# else
#  define UNUSED_PARAM(_parm) _parm
# endif
#endif

#ifndef UNUSED_TYPE
// XXX _Pragma would better but doesn't always work right now.
#  define UNUSED_TYPE(_parm) UNUSED_PARAM(_parm)
#endif

#ifndef UNUSED_VARIABLE
// XXX is there a better way?
#  define UNUSED_VARIABLE(_var) (void)_var
#endif

/*
 * gcc can warn us if we're ignoring returns
 */
#if defined(__GNUC__)
# define MUST_CHECK_RETURN __attribute__((warn_unused_result))
#else
# define MUST_CHECK_RETURN
#endif

/*
 * ALIGNED specifies minimum alignment in "n" bytes.
 */

#ifdef __GNUC__
#define ALIGNED(n) __attribute__((__aligned__(n)))
#else
#define ALIGNED(n)
#endif

/*
 * Encapsulate the syntactic differences between gcc and msvc alignment control.
 * BOUNDARY must match in the prefix and suffix.
 */

#ifdef _WIN32
#define ALIGN_PREFIX(BOUNDRY) __declspec(align(BOUNDRY))
#define ALIGN_SUFFIX(BOUNDRY)
#else
#define ALIGN_PREFIX(BOUNDRY)
#define ALIGN_SUFFIX(BOUNDRY) __attribute__((__aligned__(BOUNDRY)))
#endif

/*
 * Once upon a time, this was used to silence compiler warnings that
 * get generated when the compiler thinks that a function returns
 * when it is marked noreturn.  Don't do it.  Use NOT_REACHED().
 */

#define INFINITE_LOOP()           do { } while (1)

/*
 * On FreeBSD (for the tools build), size_t is typedef'd if _BSD_SIZE_T_
 * is defined. Use the same logic here so we don't define it twice. [greg]
 */
#ifdef __FreeBSD__
#   ifdef _BSD_SIZE_T_
#      undef _BSD_SIZE_T_
#      ifdef VM_I386
#         ifdef VM_X86_64
             typedef uint64 size_t;
#         else
             typedef uint32 size_t;
#         endif
#      endif /* VM_I386 */
#   endif

#   ifdef _BSD_SSIZE_T_
#      undef _BSD_SSIZE_T_
#      ifdef VM_I386
#         ifdef VM_X86_64
             typedef int64 ssize_t;
#         else
             typedef int32 ssize_t;
#         endif
#      endif /* VM_I386 */
#   endif

#else
#   if !defined(_SIZE_T) && !defined(_SIZE_T_DEFINED)
#      ifdef VM_I386
#         define _SIZE_T
#         ifdef VM_X86_64
             typedef uint64 size_t;
#         else
             typedef uint32 size_t;
#         endif
#      elif defined(VM_ARM_64)
#         define _SIZE_T
          typedef uint64 size_t;
#      elif defined(__arm__)
#         define _SIZE_T
          typedef uint32 size_t;
#      endif
#   endif

#   if !defined(FROBOS) && !defined(_SSIZE_T) && !defined(_SSIZE_T_) && \
       !defined(ssize_t) && !defined(__ssize_t_defined) && \
       !defined(_SSIZE_T_DECLARED) && !defined(_SSIZE_T_DEFINED) && \
       !defined(_SSIZE_T_DEFINED_)
#      ifdef VM_I386
#         define _SSIZE_T
#         define __ssize_t_defined
#         define _SSIZE_T_DECLARED
#         define _SSIZE_T_DEFINED_
#         ifdef VM_X86_64
             typedef int64 ssize_t;
#         else
             typedef int32 ssize_t;
#         endif
#      elif defined(VM_ARM_64)
#         define _SSIZE_T
#         define __ssize_t_defined
#         define _SSIZE_T_DECLARED
#         define _SSIZE_T_DEFINED_
          typedef int64 ssize_t;
#      elif defined(__arm__)
#         define _SSIZE_T
#         define __ssize_t_defined
#         define _SSIZE_T_DECLARED
#         define _SSIZE_T_DEFINED_
             typedef int32 ssize_t;
#      endif
#   endif

#endif

/*
 * Format modifier for printing pid_t.  On sun the pid_t is a ulong, but on
 * Linux it's an int.
 * Use this like this: printf("The pid is %" FMTPID ".\n", pid);
 */
#ifdef sun
#   ifdef VM_X86_64
#      define FMTPID "d"
#   else
#      define FMTPID "lu"
#   endif
#else
# define FMTPID "d"
#endif

/*
 * Format modifier for printing uid_t.  On Solaris 10 and earlier, uid_t
 * is a ulong, but on other platforms it's an unsigned int.
 * Use this like this: printf("The uid is %" FMTUID ".\n", uid);
 */
#if defined(sun) && !defined(SOL11)
#   ifdef VM_X86_64
#      define FMTUID "u"
#   else
#      define FMTUID "lu"
#   endif
#else
# define FMTUID "u"
#endif

/*
 * Format modifier for printing mode_t.  On sun the mode_t is a ulong, but on
 * Linux it's an int.
 * Use this like this: printf("The mode is %" FMTMODE ".\n", mode);
 */
#ifdef sun
#   ifdef VM_X86_64
#      define FMTMODE "o"
#   else
#      define FMTMODE "lo"
#   endif
#else
# define FMTMODE "o"
#endif

#ifdef __APPLE__
/*
 * Format specifier for all these annoying types such as {S,U}Int32
 * which are 'long' in 32-bit builds
 *       and  'int' in 64-bit builds.
 */
#   ifdef __LP64__
#      define FMTLI ""
#   else
#      define FMTLI "l"
#   endif

/*
 * Format specifier for all these annoying types such as NS[U]Integer
 * which are  'int' in 32-bit builds
 *       and 'long' in 64-bit builds.
 */
#   ifdef __LP64__
#      define FMTIL "l"
#   else
#      define FMTIL ""
#   endif
#endif

/*
 * Define type for poll device handles.
 */

typedef int64 PollDevHandle;

/*
 * Define the utf16_t type.
 */

#if defined(_WIN32) && defined(_NATIVE_WCHAR_T_DEFINED)
typedef wchar_t utf16_t;
#else
typedef uint16 utf16_t;
#endif

/*
 * Define for point and rectangle types.  Defined here so they
 * can be used by other externally facing headers in bora/public.
 */

typedef struct VMPoint {
   int x, y;
} VMPoint;

#if defined _WIN32 && defined USERLEVEL
struct tagRECT;
typedef struct tagRECT VMRect;
#else
typedef struct VMRect {
   int left;
   int top;
   int right;
   int bottom;
} VMRect;
#endif

/*
 * ranked locks "everywhere"
 */

typedef uint32 MX_Rank;

#endif  /* _VM_BASIC_TYPES_H_ */
���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vmware_pack_begin.h�������������������������������������������������������������0000444�0000000�0000000�00000002451�13432725350�017225� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmware_pack_begin.h --
 *
 *    Begin of structure packing. See vmware_pack_init.h for details.
 *
 *    Note that we do not use the following construct in this include file,
 *    because we want to emit the code every time the file is included --hpreg
 *
 *    #ifndef foo
 *    #   define foo
 *    ...
 *    #endif
 *
 */

#include "vmware_pack_init.h"

#ifdef _MSC_VER
#   pragma pack(push, 1)
#elif __GNUC__
#else
#   error Compiler packing...
#endif
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/backdoor_types.h����������������������������������������������������������������0000444�0000000�0000000�00000006773�13432725350�016605� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1999-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * backdoor_types.h --
 *
 *    Type definitions for backdoor interaction code.
 */

#ifndef _BACKDOOR_TYPES_H_
#define _BACKDOOR_TYPES_H_

#ifndef VM_I386
#error The backdoor protocol is only supported on x86 architectures.
#endif

/*
 * These #defines are intended for defining register structs as part of
 * existing named unions. If the union should encapsulate the register
 * (and nothing else), use DECLARE_REG_NAMED_STRUCT defined below.
 */

#define DECLARE_REG32_STRUCT \
   struct { \
      uint16 low; \
      uint16 high; \
   } halfs; \
   uint32 word

#define DECLARE_REG64_STRUCT \
   DECLARE_REG32_STRUCT; \
   struct { \
      uint32 low; \
      uint32 high; \
   } words; \
   uint64 quad

#if defined (VM_X86_64)  ||  defined (VM_ARM_64)
#define DECLARE_REG_STRUCT DECLARE_REG64_STRUCT
#else
#define DECLARE_REG_STRUCT DECLARE_REG32_STRUCT
#endif

#define DECLARE_REG_NAMED_STRUCT(_r) \
   union { DECLARE_REG_STRUCT; } _r

/*
 * Some of the registers are expressed by semantic name, because if they were
 * expressed as register structs declared above, we could only address them
 * by fixed size (half-word, word, quad, etc.) instead of by varying size
 * (size_t, uintptr_t).
 *
 * To be cleaner, these registers are expressed ONLY by semantic name,
 * rather than by a union of the semantic name and a register struct.
 */
typedef union {
   struct {
      DECLARE_REG_NAMED_STRUCT(ax);
      size_t size; /* Register bx. */
      DECLARE_REG_NAMED_STRUCT(cx);
      DECLARE_REG_NAMED_STRUCT(dx);
      DECLARE_REG_NAMED_STRUCT(si);
      DECLARE_REG_NAMED_STRUCT(di);
   } in;
   struct {
      DECLARE_REG_NAMED_STRUCT(ax);
      DECLARE_REG_NAMED_STRUCT(bx);
      DECLARE_REG_NAMED_STRUCT(cx);
      DECLARE_REG_NAMED_STRUCT(dx);
      DECLARE_REG_NAMED_STRUCT(si);
      DECLARE_REG_NAMED_STRUCT(di);
   } out;
} Backdoor_proto;

typedef union {
   struct {
      DECLARE_REG_NAMED_STRUCT(ax);
      DECLARE_REG_NAMED_STRUCT(bx);
      size_t size; /* Register cx. */
      DECLARE_REG_NAMED_STRUCT(dx);
      uintptr_t srcAddr; /* Register si. */
      uintptr_t dstAddr; /* Register di. */
      DECLARE_REG_NAMED_STRUCT(bp);
   } in;
   struct {
      DECLARE_REG_NAMED_STRUCT(ax);
      DECLARE_REG_NAMED_STRUCT(bx);
      DECLARE_REG_NAMED_STRUCT(cx);
      DECLARE_REG_NAMED_STRUCT(dx);
      DECLARE_REG_NAMED_STRUCT(si);
      DECLARE_REG_NAMED_STRUCT(di);
      DECLARE_REG_NAMED_STRUCT(bp);
   } out;
} Backdoor_proto_hb;

MY_ASSERTS(BACKDOOR_STRUCT_SIZES,
           ASSERT_ON_COMPILE(sizeof(Backdoor_proto) == 6 * sizeof(uintptr_t));
           ASSERT_ON_COMPILE(sizeof(Backdoor_proto_hb) == 7 * sizeof(uintptr_t));
)

#undef DECLARE_REG_STRUCT

#endif /* _BACKDOOR_TYPES_H_ */
�����vmxnet3-only/shared/compat_workqueue.h��������������������������������������������������������������0000444�0000000�0000000�00000014361�13432725347�017165� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_WORKQUEUE_H__
# define __COMPAT_WORKQUEUE_H__

#include <linux/kernel.h>

#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 5, 41)
# include <linux/workqueue.h>
#endif

/*
 *
 * Work queues and delayed work queues.
 *
 * Prior to 2.5.41, the notion of work queues did not exist.  Taskqueues are
 * used for work queues and timers are used for delayed work queues.
 *
 * After 2.6.20, normal work structs ("work_struct") and delayed work
 * ("delayed_work") structs were separated so that the work_struct could be
 * slimmed down.  The interface was also changed such that the address of the
 * work_struct itself is passed in as the argument to the work function.  This
 * requires that one embed the work struct in the larger struct containing the
 * information necessary to complete the work and use container_of() to obtain
 * the address of the containing structure.
 *
 * Users of these macros should embed a compat_work or compat_delayed_work in
 * a larger structure, then specify the larger structure as the _data argument
 * for the initialization functions, specify the work function to take
 * a compat_work_arg or compat_delayed_work_arg, then use the appropriate
 * _GET_DATA macro to obtain the reference to the structure passed in as _data.
 * An example is below.
 *
 *
 *   typedef struct WorkData {
 *      int data;
 *      compat_work work;
 *   } WorkData;
 *
 *
 *   void
 *   WorkFunc(compat_work_arg data)
 *   {
 *      WorkData *workData = COMPAT_WORK_GET_DATA(data, WorkData, work);
 *
 *      ...
 *   }
 *
 *
 *   {
 *      WorkData *workData = kmalloc(sizeof *workData, GFP_EXAMPLE);
 *      if (!workData) {
 *         return -ENOMEM;
 *      }
 *
 *      COMPAT_INIT_WORK(&workData->work, WorkFunc, workData);
 *      compat_schedule_work(&workData->work);
 *   }
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 41)  /* { */
typedef struct tq_struct compat_work;
typedef struct compat_delayed_work {
   struct tq_struct work;
   struct timer_list timer;
} compat_delayed_work;
typedef void * compat_work_arg;
typedef void * compat_delayed_work_arg;

/*
 * Delayed work queues need to run at some point in the future in process
 * context, but task queues don't support delaying the task one is scheduling.
 * Timers allow us to delay the execution of our work queue until the future,
 * but timer handlers run in bottom-half context.  As such, we use both a timer
 * and task queue and use the timer handler below to schedule the task in
 * process context immediately.  The timer lets us delay execution, and the
 * task queue lets us run in process context.
 *
 * Note that this is similar to how delayed_work is implemented with work
 * queues in later kernel versions.
 */
static inline void
__compat_delayed_work_timer(unsigned long arg)
{
   compat_delayed_work *dwork = (compat_delayed_work *)arg;
   if (dwork) {
      schedule_task(&dwork->work);
   }
}

# define COMPAT_INIT_WORK(_work, _func, _data)            \
   INIT_LIST_HEAD(&(_work)->list);                        \
   (_work)->sync = 0;                                     \
   (_work)->routine = _func;                              \
   (_work)->data = _data
# define COMPAT_INIT_DELAYED_WORK(_work, _func, _data)    \
   COMPAT_INIT_WORK(&(_work)->work, _func, _data);        \
   init_timer(&(_work)->timer);                           \
   (_work)->timer.expires = 0;                            \
   (_work)->timer.function = __compat_delayed_work_timer; \
   (_work)->timer.data = (unsigned long)_work
# define compat_schedule_work(_work)                      \
   schedule_task(_work)
# define compat_schedule_delayed_work(_work, _delay)      \
   (_work)->timer.expires = jiffies + _delay;             \
   add_timer(&(_work)->timer)
# define COMPAT_WORK_GET_DATA(_p, _type, _member)         \
   (_type *)(_p)
# define COMPAT_DELAYED_WORK_GET_DATA(_p, _type, _member) \
   (_type *)(_p)

#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20)       \
      && !defined(__VMKLNX__) /* } { */
typedef struct work_struct compat_work;
typedef struct work_struct compat_delayed_work;
typedef void * compat_work_arg;
typedef void * compat_delayed_work_arg;
# define COMPAT_INIT_WORK(_work, _func, _data)            \
   INIT_WORK(_work, _func, _data)
# define COMPAT_INIT_DELAYED_WORK(_work, _func, _data)    \
   INIT_WORK(_work, _func, _data)
# define compat_schedule_work(_work)                      \
   schedule_work(_work)
# define compat_schedule_delayed_work(_work, _delay)      \
   schedule_delayed_work(_work, _delay)
# define COMPAT_WORK_GET_DATA(_p, _type, _member)         \
   (_type *)(_p)
# define COMPAT_DELAYED_WORK_GET_DATA(_p, _type, _member) \
   (_type *)(_p)

#else  /* } Linux >= 2.6.20 { */
typedef struct work_struct compat_work;
typedef struct delayed_work compat_delayed_work;
typedef struct work_struct * compat_work_arg;
typedef struct work_struct * compat_delayed_work_arg;
# define COMPAT_INIT_WORK(_work, _func, _data)            \
   INIT_WORK(_work, _func)
# define COMPAT_INIT_DELAYED_WORK(_work, _func, _data)    \
   INIT_DELAYED_WORK(_work, _func)
# define compat_schedule_work(_work)                      \
   schedule_work(_work)
# define compat_schedule_delayed_work(_work, _delay)      \
   schedule_delayed_work(_work, _delay)
# define COMPAT_WORK_GET_DATA(_p, _type, _member)         \
   container_of(_p, _type, _member)
# define COMPAT_DELAYED_WORK_GET_DATA(_p, _type, _member) \
   container_of(_p, _type, _member.work)
#endif /* } */

#endif /* __COMPAT_WORKQUEUE_H__ */

�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/mul64.h�������������������������������������������������������������������������0000444�0000000�0000000�00000007233�13432725350�014534� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2003-2014,2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * mul64.h
 *
 *      Integer by fixed point multiplication, with rounding.
 *
 *      These routines are implemented in assembly language for most
 *      supported platforms.  This file has plain C fallback versions.
 */

#ifndef _MUL64_H_
#define _MUL64_H_

#define INCLUDE_ALLOW_USERLEVEL

#include "vm_basic_asm.h"

#if defined __cplusplus
extern "C" {
#endif

#ifdef MUL64_NO_ASM
/*
 *-----------------------------------------------------------------------------
 *
 * Mul64x3264 --
 *
 *    Unsigned integer by fixed point multiplication, with rounding:
 *       result = floor(multiplicand * multiplier * 2**(-shift) + 0.5)
 * 
 *       Unsigned 64-bit integer multiplicand.
 *       Unsigned 32-bit fixed point multiplier, represented as
 *         (multiplier, shift), where shift < 64.
 *
 * Result:
 *       Unsigned 64-bit integer product.
 *
 *-----------------------------------------------------------------------------
 */
static INLINE uint64
Mul64x3264(uint64 multiplicand, uint32 multiplier, uint32 shift)
{
   uint64 lo, hi, lo2, hi2;
   unsigned carry;

// ASSERT(shift >= 0 && shift < 64);

lo = (multiplicand & 0xffffffff) * multiplier;
   hi = (multiplicand >> 32) * multiplier;

lo2 = lo + (hi << 32);
   carry = lo2 < lo;
   hi2 = (hi >> 32) + carry;

if (shift == 0) {
      return lo2;
   } else {
      return (lo2 >> shift) + (hi2 << (64 - shift)) +
         ((lo2 >> (shift - 1)) & 1);
   }
}

// ASSERT(shift >= 0 && shift < 64);

hi = ((uint64)multiplicand >> 32) * multiplier;
   if (multiplicand < 0) {
      hi -= (uint64)multiplier << 32;
   }
   lo = ((uint64)multiplicand & 0xffffffff) * multiplier;

lo2 = lo + (hi << 32);
   carry = lo2 < lo;
   hi2 = (((int64)hi >> 32) + carry);

if (shift == 0) {
      return lo2;
   } else {
      return (lo2 >> shift) + (hi2 << (64 - shift)) +
         ((lo2 >> (shift - 1)) & 1);
   }
}
#endif

#if defined __cplusplus
} // extern "C"
#endif

#endif // _MUL64_NOASM_H_

���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_sched.h������������������������������������������������������������������0000444�0000000�0000000�00000024236�13432725347�016226� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_SCHED_H__
#   define __COMPAT_SCHED_H__

#include <linux/sched.h>

/* CLONE_KERNEL available in 2.5.35 and higher. */
#ifndef CLONE_KERNEL
#define CLONE_KERNEL CLONE_FILES | CLONE_FS | CLONE_SIGHAND
#endif

/* TASK_COMM_LEN become available in 2.6.11. */
#ifndef TASK_COMM_LEN
#define TASK_COMM_LEN 16
#endif

/* The capable() API appeared in 2.1.92 --hpreg */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 1, 92)
#   define capable(_capability) suser()
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 2, 0)
#   define need_resched() need_resched
#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 3)
#   define need_resched() (current->need_resched)
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 3)
#   define cond_resched() (need_resched() ? schedule() : (void) 0)
#endif

/* Oh well.  We need yield...  Happy us! */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 20)
#   ifdef __x86_64__
#      define compat_yield() there_is_nothing_like_yield()
#   else
#      include <linux/unistd.h>
#      include <linux/kernel.h>

/*
 * Used by _syscallX macros. Note that this is global variable, so
 * do not rely on its contents too much. As exit() is only function
 * we use, and we never check return value from exit(), we have
 * no problem...
 */
extern int errno;

/*
 * compat_exit() provides an access to the exit() function. It must 
 * be named compat_exit(), as exit() (with different signature) is 
 * provided by x86-64, arm and other (but not by i386).
 */
#      define __NR_compat_yield __NR_sched_yield
static inline _syscall0(int, compat_yield);
#   endif
#else
#   define compat_yield() yield()
#endif

/*
 * Since 2.5.34 there are two methods to enumerate tasks:
 * for_each_process(p) { ... } which enumerates only tasks and
 * do_each_thread(g,t) { ... } while_each_thread(g,t) which enumerates
 *     also threads even if they share same pid.
 */
#ifndef for_each_process
#   define for_each_process(p) for_each_task(p)
#endif

#ifndef do_each_thread
#   define do_each_thread(g, t) for_each_task(g) { t = g; do
#   define while_each_thread(g, t) while (0) }
#endif

/*
 * Lock for signal mask is moving target...
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 40) && defined(CLONE_PID)
/* 2.4.x without NPTL patches or early 2.5.x */
#define compat_sigmask_lock sigmask_lock
#define compat_dequeue_signal_current(siginfo_ptr) \
   dequeue_signal(&current->blocked, (siginfo_ptr))
#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 60) && !defined(INIT_SIGHAND)
/* RedHat's 2.4.x with first version of NPTL support, or 2.5.40 to 2.5.59 */
#define compat_sigmask_lock sig->siglock
#define compat_dequeue_signal_current(siginfo_ptr) \
   dequeue_signal(&current->blocked, (siginfo_ptr))
#else
/* RedHat's 2.4.x with second version of NPTL support, or 2.5.60+. */
#define compat_sigmask_lock sighand->siglock
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 0)
#define compat_dequeue_signal_current(siginfo_ptr) \
   dequeue_signal(&current->blocked, (siginfo_ptr))
#else
#define compat_dequeue_signal_current(siginfo_ptr) \
   dequeue_signal(current, &current->blocked, (siginfo_ptr))
#endif
#endif

/*
 * recalc_sigpending() had task argument in the past
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 29) && defined(CLONE_PID)
/* 2.4.x without NPTL patches or early 2.5.x */
#define compat_recalc_sigpending() recalc_sigpending(current)
#else
/* RedHat's 2.4.x with NPTL support, or 2.5.29+ */
#define compat_recalc_sigpending() recalc_sigpending()
#endif

/*
 * reparent_to_init() was introduced in 2.4.8.  In 2.5.38 (or possibly
 * earlier, but later than 2.5.31) a call to it was added into
 * daemonize(), so compat_daemonize no longer needs to call it.
 *
 * In 2.4.x kernels reparent_to_init() forgets to do correct refcounting
 * on current->user. It is better to count one too many than one too few...
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 8) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 38)
#define compat_reparent_to_init() do { \
					reparent_to_init(); \
					atomic_inc(&current->user->__count); \
				  } while (0)
#else
#define compat_reparent_to_init() do {} while (0)
#endif

/*
 * daemonize appeared in 2.2.18. Except 2.2.17-4-RH7.0, which has it too.
 * Fortunately 2.2.17-4-RH7.0 uses versioned symbols, so we can check
 * its existence with defined().
 */
#if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 2, 18)) && !defined(daemonize)
static inline void daemonize(void) {
   struct fs_struct *fs;

exit_mm(current);
   current->session = 1;
   current->pgrp = 1;
   exit_fs(current);
   fs = init_task.fs;
   current->fs = fs;
   atomic_inc(&fs->count);
}
#endif

/*
 * flush_signals acquires sighand->siglock since 2.5.61... Verify RH's kernels!
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 61)
#define compat_flush_signals(task) do { \
				      spin_lock_irq(&task->compat_sigmask_lock); \
				      flush_signals(task); \
				      spin_unlock_irq(&task->compat_sigmask_lock); \
				   } while (0)
#else
#define compat_flush_signals(task) flush_signals(task)
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 61)
#define compat_allow_signal(signr) do { \
                                      spin_lock_irq(&current->compat_sigmask_lock); \
                                      sigdelset(&current->blocked, signr); \
                                      compat_recalc_sigpending(); \
                                      spin_unlock_irq(&current->compat_sigmask_lock); \
                                   } while (0)
#else
#define compat_allow_signal(signr) allow_signal(signr)
#endif

/*
 * daemonize can set process name since 2.5.61. Prior to 2.5.61, daemonize
 * didn't block signals on our behalf.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 61)
#define compat_daemonize(x...)                                                \
({                                                                            \
   /* Beware! No snprintf here, so verify arguments! */                       \
   sprintf(current->comm, x);                                                 \
                                                                              \
   /* Block all signals. */                                                   \
   spin_lock_irq(&current->compat_sigmask_lock);                              \
   sigfillset(&current->blocked);                                             \
   compat_recalc_sigpending();                                                \
   spin_unlock_irq(&current->compat_sigmask_lock);                            \
   compat_flush_signals(current);                                             \
                                                                              \
   daemonize();                                                               \
   compat_reparent_to_init();                                                 \
})
#else
#define compat_daemonize(x...) daemonize(x)
#endif

/*
 * try to freeze a process. For kernels 2.6.11 or newer, we know how to choose
 * the interface. The problem is that the oldest interface, introduced in
 * 2.5.18, was backported to 2.4.x kernels. So if we're older than 2.6.11,
 * we'll decide what to do based on whether or not swsusp was configured
 * for the kernel.  For kernels 2.6.20 and newer, we'll also need to include
 * freezer.h since the try_to_freeze definition was pulled out of sched.h.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20)
#include <linux/freezer.h>
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 13) || defined(VMW_TL10S64_WORKAROUND)
#define compat_try_to_freeze() try_to_freeze()
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 11)
#define compat_try_to_freeze() try_to_freeze(PF_FREEZE)
#elif defined(CONFIG_SOFTWARE_SUSPEND) || defined(CONFIG_SOFTWARE_SUSPEND2)
#include "compat_mm.h"
#include <linux/errno.h>
#include <linux/suspend.h>
static inline int compat_try_to_freeze(void)  { 
   if (current->flags & PF_FREEZE) {
      refrigerator(PF_FREEZE); 
      return 1;
   } else {
      return 0;
   }
}
#else
static inline int compat_try_to_freeze(void) { return 0; }
#endif

/*
 * As of 2.6.23-rc1, kernel threads are no longer freezable by
 * default. Instead, kernel threads that need to be frozen must opt-in
 * by calling set_freezable() as soon as the thread is created.
 */

#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 22)
#define compat_set_freezable() do { set_freezable(); } while (0)
#else
#define compat_set_freezable() do {} while (0)
#endif

/*
 * Around 2.6.27 kernel stopped sending signals to kernel
 * threads being frozen, instead threads have to check
 * freezing() or use wait_event_freezable(). Unfortunately
 * wait_event_freezable() completely hides the fact that
 * thread was frozen from calling code and sometimes we do
 * want to know that.
 */
#ifdef PF_FREEZER_NOSIG
#define compat_wait_check_freezing() freezing(current)
#else
#define compat_wait_check_freezing() (0)
#endif

/*
 * Since 2.6.27-rc2 kill_proc() is gone... Replacement (GPL-only!)
 * API is available since 2.6.19.  Use them from 2.6.27-rc1 up.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
typedef int compat_pid;
#define compat_find_get_pid(pid) (pid)
#define compat_put_pid(pid) do { } while (0)
#define compat_kill_pid(pid, sig, flag) kill_proc(pid, sig, flag)
#else
typedef struct pid * compat_pid;
#define compat_find_get_pid(pid) find_get_pid(pid)
#define compat_put_pid(pid) put_pid(pid)
#define compat_kill_pid(pid, sig, flag) kill_pid(pid, sig, flag)
#endif

#endif /* __COMPAT_SCHED_H__ */
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_page-flags.h�������������������������������������������������������������0000444�0000000�0000000�00000005037�13432725347�017144� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_PAGE_FLAGS_H__
#   define __COMPAT_PAGE_FLAGS_H__

/* No page-flags.h prior to 2.5.12. */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 12)
#   include <linux/page-flags.h>
#endif

/* 
 * The pgoff_t type was introduced in 2.5.20, but we'll look for it by 
 * definition since it's more convenient. Note that we want to avoid a
 * situation where, in the future, a #define is changed to a typedef, 
 * so if pgoff_t is not defined in some future kernel, we won't define it.
 */
#if !defined(pgoff_t) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
#define pgoff_t unsigned long
#endif

/*
 * set_page_writeback() was introduced in 2.6.6. Prior to that, callers were
 * using the SetPageWriteback() macro directly, so that's what we'll use.
 * Prior to 2.5.12, the writeback bit didn't exist, so we don't need to do
 * anything.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 12)
#define compat_set_page_writeback(page)
#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 6)
#define compat_set_page_writeback(page) SetPageWriteback(page)
#else
#define compat_set_page_writeback(page) set_page_writeback(page)
#endif

/*
 * end_page_writeback() was introduced in 2.5.12. Prior to that, it looks like
 * there was no page writeback bit, and everything the function accomplished
 * was done by unlock_page(), so we'll define it out.
 *
 * Note that we could just #define end_page_writeback to nothing and avoid 
 * needing the compat_ prefix, but this is more complete with respect to
 * compat_set_page_writeback. 
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 12)
#define compat_end_page_writeback(page)
#else
#define compat_end_page_writeback(page) end_page_writeback(page)
#endif

#endif /* __COMPAT_PAGE_FLAGS_H__ */
�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_fs.h���������������������������������������������������������������������0000444�0000000�0000000�00000024277�13432725347�015555� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2006 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_FS_H__
#   define __COMPAT_FS_H__

#include <linux/fs.h>

/*
 * 2.6.5+ kernels define FS_BINARY_MOUNTDATA. Since it didn't exist and
 * wasn't used prior, it's safe to define it to zero.
 */

#ifndef FS_BINARY_MOUNTDATA
#define FS_BINARY_MOUNTDATA 0
#endif

/*
 * MAX_LFS_FILESIZE wasn't defined until 2.5.4.
 */
#ifndef MAX_LFS_FILESIZE
#   include <linux/pagemap.h>
#   if BITS_PER_LONG == 32
#      define MAX_LFS_FILESIZE       (((u64)PAGE_CACHE_SIZE << (BITS_PER_LONG - 1)) - 1)
#   elif BITS_PER_LONG == 64
#      define MAX_LFS_FILESIZE       0x7fffffffffffffffUL
#   endif
#endif

/*
 * sendfile as a VFS op was born in 2.5.30. Unfortunately, it also changed
 * signatures, first in 2.5.47, then again in 2.5.70, then again in 2.6.8.
 * Luckily, the 2.6.8+ signature is the same as the 2.5.47 signature.  And
 * as of 2.6.23-rc1 sendfile is gone, replaced by splice_read...
 *
 * Let's not support sendfile from 2.5.30 to 2.5.47, because the 2.5.30
 * signature is much different and file_send_actor isn't externed.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 23)
#define VMW_SENDFILE_NONE
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 8)
#define VMW_SENDFILE_NEW
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 70)
#define VMW_SENDFILE_OLD
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 47)
#define VMW_SENDFILE_NEW
#else
#define VMW_SENDFILE_NONE
#endif

/*
 * splice_read is there since 2.6.17, but let's avoid 2.6.17-rcX kernels...
 * After all nobody is using splice system call until 2.6.23 using it to
 * implement sendfile.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
#define VMW_SPLICE_READ 1
#endif

/*
 * Filesystems wishing to use generic page cache read/write routines are
 * supposed to implement aio_read and aio_write (calling into
 * generic_file_aio_read() and generic_file_aio_write() if necessary).
 *
 * The VFS exports do_sync_read() and do_sync_write() as the "new"
 * generic_file_read() and generic_file_write(), but filesystems need not
 * actually implement read and write- the VFS will automatically call
 * do_sync_write() and do_sync_read() when applications invoke the standard
 * read() and write() system calls.
 *
 * In 2.6.19, generic_file_read() and generic_file_write() were removed,
 * necessitating this change. AIO dates as far back as 2.5.42, but the API has
 * changed over time, so for simplicity, we'll only enable it from 2.6.19 and
 * on.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
# define VMW_USE_AIO
#endif

/*
 * The alloc_inode and destroy_inode VFS ops didn't exist prior to 2.4.21.
 * Without these functions, file systems can't embed inodes.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 21)
# define VMW_EMBED_INODE
#endif

/*
 * iget() was removed from the VFS as of 2.6.25-rc1. The replacement for iget()
 * is iget_locked() which was added in 2.5.17.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 17)
# define VMW_USE_IGET_LOCKED
#endif

/*
 * parent_ino was born in 2.5.5. For older kernels, let's use 2.5.5
 * implementation. It uses the dcache lock which is OK because per-dentry
 * locking appeared after 2.5.5.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 5)
#define compat_parent_ino(dentry) parent_ino(dentry)
#else
#define compat_parent_ino(dentry)                                             \
({                                                                            \
   ino_t res;                                                                 \
   spin_lock(&dcache_lock);                                                   \
   res = dentry->d_parent->d_inode->i_ino;                                    \
   spin_unlock(&dcache_lock);                                                 \
   res;                                                                       \
})
#endif

/*
 * putname changed to __putname in 2.6.6.
 */
#define compat___getname() __getname()
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 6)
#define compat___putname(name) putname(name)
#else
#define compat___putname(name) __putname(name)
#endif

/*
 * inc_nlink, drop_nlink, and clear_nlink were added in 2.6.19.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
#define compat_inc_nlink(inode) ((inode)->i_nlink++)
#define compat_drop_nlink(inode) ((inode)->i_nlink--)
#define compat_clear_nlink(inode) ((inode)->i_nlink = 0)
#else
#define compat_inc_nlink(inode) inc_nlink(inode)
#define compat_drop_nlink(inode) drop_nlink(inode)
#define compat_clear_nlink(inode) clear_nlink(inode)
#endif

/*
 * i_size_write and i_size_read were introduced in 2.6.0-test1 
 * (though we'll look for them as of 2.6.1). They employ slightly different
 * locking in order to guarantee atomicity, depending on the length of a long,
 * whether the kernel is SMP, or whether the kernel is preemptible. Prior to
 * i_size_write and i_size_read, there was no such locking, so that's the
 * behavior we'll emulate.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 1)
#define compat_i_size_read(inode) ((inode)->i_size)
#define compat_i_size_write(inode, size) ((inode)->i_size = size)
#else
#define compat_i_size_read(inode) i_size_read(inode)
#define compat_i_size_write(inode, size) i_size_write(inode, size)
#endif

/*
 * filemap_fdatawrite was introduced in 2.5.12. Prior to that, modules used
 * filemap_fdatasync instead. In 2.4.18, both filemap_fdatawrite and 
 * filemap_fdatawait began returning status codes. Prior to that, they were 
 * void functions, so we'll just have them return 0.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 18)
#define compat_filemap_fdatawrite(mapping)                                    \
({                                                                            \
   int result = 0;                                                            \
   filemap_fdatasync(mapping);                                                \
   result;                                                                    \
})
#define compat_filemap_fdatawait(mapping)                                     \
({                                                                            \
   int result = 0;                                                            \
   filemap_fdatawait(mapping);                                                \
   result;                                                                    \
})
#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 12)
#define compat_filemap_fdatawrite(mapping) filemap_fdatasync(mapping)
#define compat_filemap_fdatawait(mapping) filemap_fdatawait(mapping)
#else
#define compat_filemap_fdatawrite(mapping) filemap_fdatawrite(mapping)
#define compat_filemap_fdatawait(mapping) filemap_fdatawait(mapping)
#endif

/*
 * filemap_write_and_wait was introduced in 2.6.6 and exported for module use
 * in 2.6.16. It's really just a simple wrapper around filemap_fdatawrite and 
 * and filemap_fdatawait, which initiates a flush of all dirty pages, then 
 * waits for the pages to flush. The implementation here is a simplified form 
 * of the one found in 2.6.20-rc3.
 *
 * Unfortunately, it just isn't possible to implement this prior to 2.4.5, when
 * neither filemap_fdatawait nor filemap_fdatasync were exported for module
 * use. So we'll define it out and hope for the best.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 5)
#define compat_filemap_write_and_wait(mapping)
#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 16)
#define compat_filemap_write_and_wait(mapping)                                \
({                                                                            \
   int result = 0;                                                            \
   if (mapping->nrpages) {                                                    \
      result = compat_filemap_fdatawrite(mapping);                            \
      if (result != -EIO) {                                                   \
         int result2 = compat_filemap_fdatawait(mapping);                     \
         if (!result) {                                                       \
            result = result2;                                                 \
         }                                                                    \
      }                                                                       \
   }                                                                          \
   result;                                                                    \
})
#else
#define compat_filemap_write_and_wait(mapping) filemap_write_and_wait(mapping)
#endif

/*
 * invalidate_remote_inode was introduced in 2.6.0-test5. Prior to that, 
 * filesystems wishing to invalidate pages belonging to an inode called 
 * invalidate_inode_pages.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 0)
#define compat_invalidate_remote_inode(inode) invalidate_inode_pages(inode)
#else
#define compat_invalidate_remote_inode(inode) invalidate_remote_inode(inode)
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
#define VMW_FSYNC_OLD
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 3, 0)
typedef umode_t compat_umode_t;
#else
typedef int compat_umode_t;
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
#define d_make_root(inode) ({                      \
   struct dentry * ____res = d_alloc_root(inode);  \
   if (!____res) {                                 \
      iput(inode);                                 \
   }                                               \
   ____res;                                        \
})
#endif
#endif /* __COMPAT_FS_H__ */
���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/driverLog.c���������������������������������������������������������������������0000444�0000000�0000000�00000011117�13432725347�015517� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007-2014 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * driverLog.c --
 *
 *      Common logging functions for Linux kernel modules.
 */

#include "driver-config.h"
#include "compat_kernel.h"
#include "compat_sched.h"
#include <asm/current.h>

#include "driverLog.h"

#define LINUXLOG_BUFFER_SIZE 1024

static const char *driverLogPrefix = "";

/*
 * vsnprintf was born in 2.4.10. Fall back on vsprintf if we're
 * an older kernel.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 10)
# define vsnprintf(str, size, fmt, args) vsprintf(str, fmt, args)
#endif

/*
 *----------------------------------------------------------------------------
 *
 * DriverLog_Init --
 *
 *      Initializes the Linux logging.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------------
 */

void
DriverLog_Init(const char *prefix) // IN
{
   driverLogPrefix = prefix ? prefix : "";
}

/*
 *----------------------------------------------------------------------
 *
 * DriverLogPrint --
 *
 *      Log error message from a Linux module.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static void
DriverLogPrint(const char *level,     // IN: KERN_* constant
               const char *fmt,       // IN: error format string
               va_list args)          // IN: arguments for format string
{
   static char staticBuf[LINUXLOG_BUFFER_SIZE];
   char stackBuf[128];
   va_list args2;
   const char *buf;

/*
    * By default, use a small buffer on the stack (thread safe). If it is too
    * small, fall back to a larger static buffer (not thread safe).
    */
   va_copy(args2, args);
   if (vsnprintf(stackBuf, sizeof stackBuf, fmt, args2) < sizeof stackBuf) {
      buf = stackBuf;
   } else {
      vsnprintf(staticBuf, sizeof staticBuf, fmt, args);
      buf = staticBuf;
   }
   va_end(args2);

printk("%s%s[%d]: %s", level, driverLogPrefix, current->pid, buf);
}

/*
 *----------------------------------------------------------------------
 *
 * Warning --
 *
 *      Warning messages from kernel module: logged into kernel log
 *      as warnings.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

void
Warning(const char *fmt, ...)  // IN: warning format string
{
   va_list args;

va_start(args, fmt);
   DriverLogPrint(KERN_WARNING, fmt, args);
   va_end(args);
}

/*
 *----------------------------------------------------------------------
 *
 * Log --
 *
 *      Log messages from kernel module: logged into kernel log
 *      as debug information.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

void
Log(const char *fmt, ...)  // IN: log format string
{
   va_list args;

/*
    * Use the kernel log with at least a KERN_DEBUG level
    * so it doesn't garbage the screen at (re)boot time on RedHat 6.0.
    */

va_start(args, fmt);
   DriverLogPrint(KERN_DEBUG, fmt, args);
   va_end(args);
}

/*
 *----------------------------------------------------------------------
 *
 * Panic --
 *
 *      ASSERTION failures and Panics from kernel module get here.
 *      Message is logged to the kernel log and on console.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      Never returns
 *
 *----------------------------------------------------------------------
 */

void
Panic(const char *fmt, ...)  // IN: panic format string
{
   va_list args;

va_start(args, fmt);
   DriverLogPrint(KERN_EMERG, fmt, args);
   va_end(args);

#ifdef BUG
   BUG();
#else
   /* Should die with %cs unwritable, or at least with page fault. */
   asm volatile("movb $0, %cs:(0)");
#endif

while (1);
}
�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_autoconf.h���������������������������������������������������������������0000444�0000000�0000000�00000002641�13432725347�016752� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2009 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_AUTOCONF_H__
#   define __COMPAT_AUTOCONF_H__

#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMKDRIVERS
#include "includeCheck.h"

#ifndef LINUX_VERSION_CODE
#   error "Include compat_version.h before compat_autoconf.h"
#endif

/* autoconf.h moved from linux/autoconf.h to generated/autoconf.h in 2.6.33-rc1. */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 33)
#   include <linux/autoconf.h>
#else
#   include <generated/autoconf.h>
#endif

#endif /* __COMPAT_AUTOCONF_H__ */
�����������������������������������������������������������������������������������������������vmxnet3-only/shared/npa_defs.h����������������������������������������������������������������������0000444�0000000�0000000�00000004612�13432725350�015342� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2009-2014 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef _NPA_DEFS_H
#define _NPA_DEFS_H

#define NPA_PLUGIN_NUMPAGES      64
#define NPA_MEMIO_NUMPAGES       32
#define NPA_SHARED_NUMPAGES      6
#define VMXNET3_NPA_CMD_SUCCESS   1
#define VMXNET3_NPA_CMD_FAILURE   0
#define NPA_MAX_PLUGINS_PER_VM   4
#define VMXNET3_PLUGIN_INFO_LEN  32

/* these structure are versioned using the vmxnet3 version */

typedef 
#include "vmware_pack_begin.h"
struct NPA_PluginPages {
   uint64 vaddr;
   uint32 numPages;
   uint32 _pad;
   PPN64  pages[NPA_PLUGIN_NUMPAGES * NPA_MAX_PLUGINS_PER_VM];
}
#include "vmware_pack_end.h"
NPA_PluginPages;

typedef
#include "vmware_pack_begin.h"
struct NPA_MemioPages {
   PPN64  startPPN;
   uint32 numPages;
   uint32 _pad;
}
#include "vmware_pack_end.h"
NPA_MemioPages;

typedef
#include "vmware_pack_begin.h"
struct NPA_SharedPages {
   PPN64  startPPN;
   uint32 numPages;
   uint32 _pad;
}
#include "vmware_pack_end.h"
NPA_SharedPages;

typedef
#include "vmware_pack_begin.h"
struct NPA_PluginConf {
   NPA_PluginPages   pluginPages;
   NPA_MemioPages    memioPages;
   NPA_SharedPages   sharedPages;
   uint64            entryVA;    // address of entry function in the plugin 
   uint32            deviceInfo[VMXNET3_PLUGIN_INFO_LEN]; // opaque data returned by PF driver
   uint64            _reserved;  // give room for an extra pointer
}
#include "vmware_pack_end.h"
NPA_PluginConf;

/* vmkernel and device backend shared definitions */

#define VMXNET3_PLUGIN_NAME_LEN  256
#define VMXNET3_PLUGIN_REPOSITORY "/usr/lib/vmware/npa_plugins"
#define NPA_MEMIO_REGIONS_MAX    6

#endif // _NPA_DEFS_H
����������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_skbuff.h�����������������������������������������������������������������0000444�0000000�0000000�00000016131�13432725347�016413� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_SKBUFF_H__
#   define __COMPAT_SKBUFF_H__

#include <linux/skbuff.h>

/*
 * When transition from mac/nh/h to skb_* accessors was made, also SKB_WITH_OVERHEAD
 * was introduced.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22) || \
   (LINUX_VERSION_CODE == KERNEL_VERSION(2, 6, 21) && defined(SKB_WITH_OVERHEAD))
#define compat_skb_mac_header(skb)         skb_mac_header(skb)
#define compat_skb_network_header(skb)     skb_network_header(skb)
#define compat_skb_network_offset(skb)     skb_network_offset(skb)
#define compat_skb_transport_header(skb)   skb_transport_header(skb)
#define compat_skb_transport_offset(skb)   skb_transport_offset(skb)
#define compat_skb_network_header_len(skb) skb_network_header_len(skb)
#define compat_skb_tail_pointer(skb)       skb_tail_pointer(skb)
#define compat_skb_end_pointer(skb)        skb_end_pointer(skb)
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
#   define compat_skb_ip_header(skb)       ip_hdr(skb)
#   define compat_skb_ipv6_header(skb)     ipv6_hdr(skb)
#   define compat_skb_tcp_header(skb)      tcp_hdr(skb)
#else
#   define compat_skb_ip_header(skb)   ((struct iphdr *)skb_network_header(skb))
#   define compat_skb_ipv6_header(skb) ((struct ipv6hdr *)skb_network_header(skb))
#   define compat_skb_tcp_header(skb)  ((struct tcphdr *)skb_transport_header(skb))
#endif
#define compat_skb_reset_mac_header(skb)          skb_reset_mac_header(skb)
#define compat_skb_reset_network_header(skb)      skb_reset_network_header(skb)
#define compat_skb_reset_transport_header(skb)    skb_reset_transport_header(skb)
#define compat_skb_set_network_header(skb, off)   skb_set_network_header(skb, off)
#define compat_skb_set_transport_header(skb, off) skb_set_transport_header(skb, off)
#else
#define compat_skb_mac_header(skb)         (skb)->mac.raw
#define compat_skb_network_header(skb)     (skb)->nh.raw
#define compat_skb_network_offset(skb)     ((skb)->nh.raw - (skb)->data)
#define compat_skb_transport_header(skb)   (skb)->h.raw
#define compat_skb_transport_offset(skb)   ((skb)->h.raw - (skb)->data)
#define compat_skb_network_header_len(skb) ((skb)->h.raw - (skb)->nh.raw)
#define compat_skb_tail_pointer(skb)       (skb)->tail
#define compat_skb_end_pointer(skb)        (skb)->end
#define compat_skb_ip_header(skb)          (skb)->nh.iph
#define compat_skb_ipv6_header(skb)        (skb)->nh.ipv6h
#define compat_skb_tcp_header(skb)         (skb)->h.th
#define compat_skb_reset_mac_header(skb)   ((skb)->mac.raw = (skb)->data)
#define compat_skb_reset_network_header(skb)      ((skb)->nh.raw = (skb)->data)
#define compat_skb_reset_transport_header(skb)    ((skb)->h.raw = (skb)->data)
#define compat_skb_set_network_header(skb, off)   ((skb)->nh.raw = (skb)->data + (off))
#define compat_skb_set_transport_header(skb, off) ((skb)->h.raw = (skb)->data + (off))
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18) || defined(VMW_SKB_LINEARIZE_2618)
#   define compat_skb_linearize(skb) skb_linearize((skb))
#else

#   if LINUX_VERSION_CODE == KERNEL_VERSION(2, 6, 0)
#      define compat_skb_linearize(skb) __skb_linearize((skb), GFP_ATOMIC)
#   elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 4)
#      define compat_skb_linearize(skb) skb_linearize((skb), GFP_ATOMIC)
#   else
static inline int
compat_skb_linearize(struct sk_buff *skb)
{
   return 0;
}
#   endif

#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20)
#define compat_skb_csum_offset(skb)        (skb)->csum_offset
#else
#define compat_skb_csum_offset(skb)        (skb)->csum
#endif

/*
 * Note that compat_skb_csum_start() has semantic different from kernel's csum_start:
 * kernel's skb->csum_start is offset between start of checksummed area and start of
 * complete skb buffer, while our compat_skb_csum_start(skb) is offset from start
 * of packet itself.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 22)
#define compat_skb_csum_start(skb)         ((skb)->csum_start - skb_headroom(skb))
#else
#define compat_skb_csum_start(skb)         compat_skb_transport_offset(skb)
#endif

#if defined(NETIF_F_GSO) /* 2.6.18 and upwards */
#define compat_skb_mss(skb) (skb_shinfo(skb)->gso_size)
#else
#define compat_skb_mss(skb) (skb_shinfo(skb)->tso_size)
#endif

/* used by both received pkts and outgoing ones */
#define VM_CHECKSUM_UNNECESSARY CHECKSUM_UNNECESSARY

/* csum status of received pkts */
#if defined(CHECKSUM_COMPLETE)
#   define VM_RX_CHECKSUM_PARTIAL     CHECKSUM_COMPLETE
#elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19) && defined(CHECKSUM_HW)
#   define VM_RX_CHECKSUM_PARTIAL     CHECKSUM_HW
#else
#   define VM_RX_CHECKSUM_PARTIAL     CHECKSUM_PARTIAL
#endif

/* csum status of outgoing pkts */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19) && defined(CHECKSUM_HW)
#   define VM_TX_CHECKSUM_PARTIAL      CHECKSUM_HW
#else
#   define VM_TX_CHECKSUM_PARTIAL      CHECKSUM_PARTIAL
#endif

#if (LINUX_VERSION_CODE < KERNEL_VERSION(2,1,0))
#   define compat_kfree_skb(skb, type) kfree_skb(skb, type)
#   define compat_dev_kfree_skb(skb, type) dev_kfree_skb(skb, type)
#   define compat_dev_kfree_skb_any(skb, type) dev_kfree_skb(skb, type)
#   define compat_dev_kfree_skb_irq(skb, type) dev_kfree_skb(skb, type)
#else
#   define compat_kfree_skb(skb, type) kfree_skb(skb)
#   define compat_dev_kfree_skb(skb, type) dev_kfree_skb(skb)
#   if (LINUX_VERSION_CODE < KERNEL_VERSION(2,3,43))
#      define compat_dev_kfree_skb_any(skb, type) dev_kfree_skb(skb)
#      define compat_dev_kfree_skb_irq(skb, type) dev_kfree_skb(skb)
#   else
#      define compat_dev_kfree_skb_any(skb, type) dev_kfree_skb_any(skb)
#      define compat_dev_kfree_skb_irq(skb, type) dev_kfree_skb_irq(skb)
#   endif
#endif

#ifndef NET_IP_ALIGN
#   define COMPAT_NET_IP_ALIGN  2
#else
#   define COMPAT_NET_IP_ALIGN  NET_IP_ALIGN 
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 4, 4)
#   define compat_skb_headlen(skb)         skb_headlen(skb)
#   define compat_pskb_may_pull(skb, len)  pskb_may_pull(skb, len)
#   define compat_skb_is_nonlinear(skb)    skb_is_nonlinear(skb)
#else
#   define compat_skb_headlen(skb)         (skb)->len
#   define compat_pskb_may_pull(skb, len)  1
#   define compat_skb_is_nonlinear(skb)    0
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 12)
#   define compat_skb_header_cloned(skb)   skb_header_cloned(skb)
#else
#   define compat_skb_header_cloned(skb)   0
#endif
#endif /* __COMPAT_SKBUFF_H__ */
���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_semaphore.h��������������������������������������������������������������0000444�0000000�0000000�00000003142�13432725347�017114� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_SEMAPHORE_H__
#   define __COMPAT_SEMAPHORE_H__

/* <= 2.6.25 have asm only, 2.6.26 has both, and 2.6.27-rc2+ has linux only. */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
#   include <asm/semaphore.h>
#else
#   include <linux/semaphore.h>
#endif

/*
* The init_MUTEX_LOCKED() API appeared in 2.2.18, and is also in
* 2.2.17-21mdk --hpreg
*/

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 2, 18)
   #ifndef init_MUTEX_LOCKED
      #define init_MUTEX_LOCKED(_sem) *(_sem) = MUTEX_LOCKED
   #endif
   #ifndef DECLARE_MUTEX
      #define DECLARE_MUTEX(name) struct semaphore name = MUTEX
   #endif
   #ifndef DECLARE_MUTEX_LOCKED
      #define DECLARE_MUTEX_LOCKED(name) struct semaphore name = MUTEX_LOCKED
   #endif
#endif

#endif /* __COMPAT_SEMAPHORE_H__ */
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vmware_pack_end.h���������������������������������������������������������������0000444�0000000�0000000�00000002475�13432725350�016715� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmware_pack_end.h --
 *
 *    End of structure packing. See vmware_pack_init.h for details.
 *
 *    Note that we do not use the following construct in this include file,
 *    because we want to emit the code every time the file is included --hpreg
 *
 *    #ifndef foo
 *    #   define foo
 *    ...
 *    #endif
 *
 */

#include "vmware_pack_init.h"

#ifdef _MSC_VER
#   pragma pack(pop)
#elif __GNUC__
__attribute__((__packed__))
#else
#   error Compiler packing...
#endif
���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_uaccess.h����������������������������������������������������������������0000444�0000000�0000000�00000006062�13432725347�016563� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_UACCESS_H__
#   define __COMPAT_UACCESS_H__

/* User space access functions moved in 2.1.7 to asm/uaccess.h --hpreg */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 1, 7)
#   include <asm/uaccess.h>
#else
#   include <asm/segment.h>
#endif

/* get_user() API modified in 2.1.4 to take 2 arguments --hpreg */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 1, 4)
#   define compat_get_user get_user
#else
/*
 * We assign 0 to the variable in case of failure to prevent "`_var' might be
 * used uninitialized in this function" compiler warnings. I think it is OK,
 * because the hardware-based version in newer kernels probably has the same
 * semantics and does not guarantee that the value of _var will not be
 * modified, should the access fail --hpreg
 */
#   define compat_get_user(_var, _uvAddr) ({                        \
   int _status;                                                     \
                                                                    \
   _status = verify_area(VERIFY_READ, _uvAddr, sizeof(*(_uvAddr))); \
   if (_status == 0) {                                              \
      (_var) = get_user(_uvAddr);                                   \
   } else {                                                         \
      (_var) = 0;                                                   \
   }                                                                \
   _status;                                                         \
})
#endif

/*
 * The copy_from_user() API appeared in 2.1.4
 *
 * The emulation is not perfect here, but it is conservative: on failure, we
 * always return the total size, instead of the potentially smaller faulty
 * size --hpreg
 *
 * Since 2.5.55 copy_from_user() is no longer macro.
 */
#if !defined(copy_from_user) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 2, 0)
#   define copy_from_user(_to, _from, _size) ( \
   verify_area(VERIFY_READ, _from, _size)      \
       ? (_size)                               \
       : (memcpy_fromfs(_to, _from, _size), 0) \
)
#   define copy_to_user(_to, _from, _size) ( \
   verify_area(VERIFY_WRITE, _to, _size)     \
       ? (_size)                             \
       : (memcpy_tofs(_to, _from, _size), 0) \
)
#endif

#endif /* __COMPAT_UACCESS_H__ */
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_spinlock.h���������������������������������������������������������������0000444�0000000�0000000�00000003377�13432725347�016765� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2005 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_SPINLOCK_H__
#   define __COMPAT_SPINLOCK_H__

#include <linux/spinlock.h>

/*
 * Preempt support was added during 2.5.x development cycle, and later
 * it was backported to 2.4.x.  In 2.4.x backport these definitions
 * live in linux/spinlock.h, that's why we put them here (in 2.6.x they
 * are defined in linux/preempt.h which is included by linux/spinlock.h).
 */
#ifdef CONFIG_PREEMPT
#define compat_preempt_disable() preempt_disable()
#define compat_preempt_enable()  preempt_enable()
#else
#define compat_preempt_disable() do { } while (0)
#define compat_preempt_enable()  do { } while (0)
#endif

/* Some older kernels - 2.6.10 and earlier - lack DEFINE_SPINLOCK */
#ifndef DEFINE_SPINLOCK
#define DEFINE_SPINLOCK(x) spinlock_t x = SPIN_LOCK_UNLOCKED
#endif

/* Same goes for DEFINE_RWLOCK */
#ifndef DEFINE_RWLOCK
#define DEFINE_RWLOCK(x)   rwlock_t x = RW_LOCK_UNLOCKED
#endif

#endif /* __COMPAT_SPINLOCK_H__ */
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/driverLog.h���������������������������������������������������������������������0000444�0000000�0000000�00000002237�13432725347�015527� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007-2014 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * driverLog.h --
 *
 *      Logging functions for Linux kernel modules.
 */

#ifndef __DRIVERLOG_H__
#define __DRIVERLOG_H__

/*
 * The definitions of Warning(), Log(), and Panic() come from vm_assert.h for
 * consistency.
 */
#include "vm_assert.h"

void DriverLog_Init(const char *prefix);

#endif /* __DRIVERLOG_H__ */
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/x86cpuid.h����������������������������������������������������������������������0000644�0000000�0000000�00000327332�13432725350�015246� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef _X86CPUID_H_
#define _X86CPUID_H_

/* http://www.sandpile.org/ia32/cpuid.htm */

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_VMX

#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_VMMON
#include "includeCheck.h"

#include "vm_basic_types.h"
#include "community_source.h"
#include "x86vendor.h"
#include "vm_assert.h"

#if defined __cplusplus
extern "C" {
#endif

/*
 * The linux kernel's ptrace.h stupidly defines the bare
 * EAX/EBX/ECX/EDX, which wrecks havoc with our preprocessor tricks.
 */
#undef EAX
#undef EBX
#undef ECX
#undef EDX

typedef struct CPUIDRegs {
   uint32 eax, ebx, ecx, edx;
} CPUIDRegs;

typedef union CPUIDRegsUnion {
   uint32 array[4];
   CPUIDRegs regs;
} CPUIDRegsUnion;

/*
 * Results of calling cpuid(eax, ecx) on all host logical CPU.
 */
#ifdef _MSC_VER
// TODO: Move this under the push
#pragma warning (disable :4200) // non-std extension: zero-sized array in struct
#pragma warning (push)
#pragma warning (disable :4100) // unreferenced parameters
#endif

typedef
#include "vmware_pack_begin.h"
struct CPUIDReply {
   /*
    * Unique host logical CPU identifier. It does not change across queries, so
    * we use it to correlate the replies of multiple queries.
    */
   uint64 tag;                // OUT

CPUIDRegs regs;            // OUT
}
#include "vmware_pack_end.h"
CPUIDReply;

typedef
#include "vmware_pack_begin.h"
struct CPUIDQuery {
   uint32 eax;                // IN
   uint32 ecx;                // IN
   uint32 numLogicalCPUs;     // IN/OUT
   CPUIDReply logicalCPUs[0]; // OUT
}
#include "vmware_pack_end.h"
CPUIDQuery;

/*
 * CPUID levels the monitor caches.
 *
 * The first parameter defines whether the level has its default masks
 * generated from the values in this file. Any level which is marked as FALSE
 * here *must* have all monitor support types set to NA. A static assert in
 * lib/cpuidcompat/cpuidcompat.c will check this.
 *
 * The second parameter is the "short name" of the level. It's mainly used for
 * token concatenation in various macros.
 *
 * The third parameter is the actual numeric value of that level (the EAX input
 * value).
 *
 * The fourth parameter is a "subleaf count", where 0 means that ecx is
 * ignored, otherwise is the count of sub-leaves.
 *
 * The fifth parameter is the first hardware version that is *aware* of the
 * CPUID level (0 = existed since dawn of time), even though we may not expose
 * this level or parts of it to guest.
 */

#define CPUID_CACHED_LEVELS                         \
   CPUIDLEVEL(TRUE,  0,   0,          0,  0)        \
   CPUIDLEVEL(TRUE,  1,   1,          0,  0)        \
   CPUIDLEVEL(FALSE, 2,   2,          0,  0)        \
   CPUIDLEVEL(FALSE, 4,   4,          7,  0)        \
   CPUIDLEVEL(FALSE, 5,   5,          0,  0)        \
   CPUIDLEVEL(TRUE,  6,   6,          0,  0)        \
   CPUIDLEVEL(TRUE,  7,   7,          1,  0)        \
   CPUIDLEVEL(FALSE, A,   0xA,        0,  0)        \
   CPUIDLEVEL(FALSE, B,   0xB,        2,  0)        \
   CPUIDLEVEL(TRUE,  D,   0xD,       10,  0)        \
   CPUIDLEVEL(TRUE,  F,   0xF,        2, 13)        \
   CPUIDLEVEL(TRUE,  10,  0x10,       2, 13)        \
   CPUIDLEVEL(TRUE,  12,  0x12,       4, 13)        \
   CPUIDLEVEL(TRUE,  14,  0x14,       2, 13)        \
   CPUIDLEVEL(TRUE,  15,  0x15,       0, 13)        \
   CPUIDLEVEL(TRUE,  16,  0x16,       0, 13)        \
   CPUIDLEVEL(TRUE,  17,  0x17,       4, 14)        \
   CPUIDLEVEL(FALSE, 400, 0x40000000, 0,  0)        \
   CPUIDLEVEL(FALSE, 401, 0x40000001, 0,  0)        \
   CPUIDLEVEL(FALSE, 402, 0x40000002, 0,  0)        \
   CPUIDLEVEL(FALSE, 403, 0x40000003, 0,  0)        \
   CPUIDLEVEL(FALSE, 404, 0x40000004, 0,  0)        \
   CPUIDLEVEL(FALSE, 405, 0x40000005, 0,  0)        \
   CPUIDLEVEL(FALSE, 406, 0x40000006, 0,  0)        \
   CPUIDLEVEL(FALSE, 410, 0x40000010, 0,  0)        \
   CPUIDLEVEL(FALSE, 80,  0x80000000, 0,  0)        \
   CPUIDLEVEL(TRUE,  81,  0x80000001, 0,  0)        \
   CPUIDLEVEL(FALSE, 82,  0x80000002, 0,  0)        \
   CPUIDLEVEL(FALSE, 83,  0x80000003, 0,  0)        \
   CPUIDLEVEL(FALSE, 84,  0x80000004, 0,  0)        \
   CPUIDLEVEL(FALSE, 85,  0x80000005, 0,  0)        \
   CPUIDLEVEL(FALSE, 86,  0x80000006, 0,  0)        \
   CPUIDLEVEL(FALSE, 87,  0x80000007, 0,  0)        \
   CPUIDLEVEL(TRUE,  88,  0x80000008, 0,  0)        \
   CPUIDLEVEL(TRUE,  8A,  0x8000000A, 0,  0)        \
   CPUIDLEVEL(FALSE, 819, 0x80000019, 0,  0)        \
   CPUIDLEVEL(FALSE, 81A, 0x8000001A, 0,  0)        \
   CPUIDLEVEL(FALSE, 81B, 0x8000001B, 0,  0)        \
   CPUIDLEVEL(FALSE, 81C, 0x8000001C, 0,  0)        \
   CPUIDLEVEL(FALSE, 81D, 0x8000001D, 5,  0)        \
   CPUIDLEVEL(FALSE, 81E, 0x8000001E, 0,  0)        \
   CPUIDLEVEL(TRUE,  81F, 0x8000001F, 0, 14)

#define CPUID_ALL_LEVELS CPUID_CACHED_LEVELS

/* Define cached CPUID levels in the form: CPUID_LEVEL_<ShortName> */
typedef enum {
#define CPUIDLEVEL(t, s, v, c, h) CPUID_LEVEL_##s,
   CPUID_CACHED_LEVELS
#undef CPUIDLEVEL
   CPUID_NUM_CACHED_LEVELS
} CpuidCachedLevel;

/* Enum to translate between shorthand name and actual CPUID level value. */
enum {
#define CPUIDLEVEL(t, s, v, c, h) CPUID_LEVEL_VAL_##s = v,
   CPUID_ALL_LEVELS
#undef CPUIDLEVEL
};

/* Named feature leaves */
#define CPUID_FEATURE_INFORMATION  0x01
#define CPUID_PROCESSOR_TOPOLOGY   4
#define CPUID_MWAIT_FEATURES       5
#define CPUID_XSAVE_FEATURES       0xd
#define CPUID_SGX_FEATURES         0x12
#define CPUID_PT_FEATURES          0x14
#define CPUID_HYPERVISOR_LEVEL_0   0x40000000
#define CPUID_SVM_FEATURES         0x8000000a

/*
 * CPUID result registers
 */

#define CPUID_REGS                              \
   CPUIDREG(EAX, eax)                           \
   CPUIDREG(EBX, ebx)                           \
   CPUIDREG(ECX, ecx)                           \
   CPUIDREG(EDX, edx)

typedef enum {
#define CPUIDREG(uc, lc) CPUID_REG_##uc,
   CPUID_REGS
#undef CPUIDREG
   CPUID_NUM_REGS
} CpuidReg;

#define CPUID_INTEL_VENDOR_STRING       "GenuntelineI"
#define CPUID_AMD_VENDOR_STRING         "AuthcAMDenti"
#define CPUID_CYRIX_VENDOR_STRING       "CyriteadxIns"
#define CPUID_VIA_VENDOR_STRING         "CentaulsaurH"
#define CPUID_HYGON_VENDOR_STRING       "HygouinenGen"

#define CPUID_HYPERV_HYPERVISOR_VENDOR_STRING  "Microsoft Hv"
#define CPUID_KVM_HYPERVISOR_VENDOR_STRING     "KVMKVMKVM\0\0\0"
#define CPUID_VMWARE_HYPERVISOR_VENDOR_STRING  "VMwareVMware"
#define CPUID_XEN_HYPERVISOR_VENDOR_STRING     "XenVMMXenVMM"

#define CPUID_INTEL_VENDOR_STRING_FIXED "GenuineIntel"
#define CPUID_AMD_VENDOR_STRING_FIXED   "AuthenticAMD"
#define CPUID_CYRIX_VENDOR_STRING_FIXED "CyrixInstead"
#define CPUID_VIA_VENDOR_STRING_FIXED   "CentaurHauls"
#define CPUID_HYGON_VENDOR_STRING_FIXED "HygonGenuine"

/*
 * FIELD can be defined to process the CPUID information provided in the
 * following CPUID_FIELD_DATA macro.
 *
 * The first parameter is the CPUID level of the feature (must be defined in
 * CPUID_ALL_LEVELS, above).
 *
 * The second parameter is the CPUID sub-level (subleaf) of the feature. Please
 * make sure here the number is consistent with the "subleaf count" in
 * CPUIDLEVEL macro. I.e., if a feature is being added to a _new_ subleaf,
 * update the subleaf count above as well.
 *
 * The third parameter is the result register.
 *
 * The fourth and fifth parameters are the bit position of the field and the
 * width, respectively.
 *
 * The sixth is the name of the field.
 *
 * The seventh parameter specifies the monitor support characteristics for
 * this field. The value must be a valid CpuidFieldSupported value (omitting
 * CPUID_FIELD_SUPPORT_ for convenience). The meaning of those values are
 * described below.
 *
 * The eighth parameter specifies the first virtual hardware version that
 * implements the field (if 7th field is YES or ANY), or 0 (if 7th field is
 * NO or NA).  The field's hardware version must match the version in
 * defaultMasks (cpuidcompat.c) if defined there, and must be less than or
 * equal to the version of the cpuid leaf it's in.
 *
 * The ninth parameter describes whether the feature is capable of being used
 * by usermode code (TRUE), or just CPL0 kernel code (FALSE).
 *
 * FLAG is defined identically to FIELD, but its accessors are more appropriate
 * for 1-bit flags, and compile-time asserts enforce that the size is 1 bit
 * wide.
 */

/*
 * CpuidFieldSupported is made up of the following values:
 *
 *     NO: A feature/field that IS NOT SUPPORTED by the monitor.  Even
 *     if the host supports this feature, we will never expose it to
 *     the guest.
 *
 *     YES: A feature/field that IS SUPPORTED by the monitor.  If the
 *     host supports this feature, we will expose it to the guest.  If
 *     not, then we will not set the feature.
 *
 *     ANY: A feature/field that IS ALWAYS SUPPORTED by the monitor.
 *     Even if the host does not support the feature, the monitor can
 *     expose the feature to the guest. As with "YES", the guest cpuid
 *     value defaults to the host/evc cpuid value.  But usually the
 *     guest cpuid value is recomputed at power on, ignoring the default
 *     value.
 *
 *
 *     NA: Only legal for levels not masked/tested by default (see
 *     above for this definition).  Such fields must always be marked
 *     as NA.
 *
 * These distinctions, when combined with the feature's CPL3
 * properties can be translated into a common CPUID mask string as
 * follows:
 *
 *     NO + CPL3 --> "R" (Reserved).  We don't support the feature,
 *     but we can't properly hide this from applications when using
 *     direct execution or HV with apps that do try/catch/fail, so we
 *     must still perform compatibility checks.
 *
 *     NO + !CPL3 --> "0" (Masked).  We can hide this from the guest.
 *
 *     YES --> "H" (Host).  We support the feature, so show it to the
 *     guest if the host has the feature.
 *
 *     ANY/NA --> "X" (Ignore).  By default, don't perform checks for
 *     this feature bit.  Per-GOS masks may choose to set this bit in
 *     the guest.  (e.g. the APIC feature bit is always set to 1.)
 *
 *     See lib/cpuidcompat/cpuidcompat.c for any possible overrides to
 *     these defaults.
 */
typedef enum {
   CPUID_FIELD_SUPPORTED_NO,
   CPUID_FIELD_SUPPORTED_YES,
   CPUID_FIELD_SUPPORTED_ANY,
   CPUID_FIELD_SUPPORTED_NA,
   CPUID_NUM_FIELD_SUPPORTEDS
} CpuidFieldSupported;

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_0                                               \
FIELD(  0,  0, EAX,  0, 32, NUMLEVELS,                         ANY,  4, FALSE) \
FIELD(  0,  0, EBX,  0, 32, VENDOR1,                           YES,  4, TRUE)  \
FIELD(  0,  0, ECX,  0, 32, VENDOR3,                           YES,  4, TRUE)  \
FIELD(  0,  0, EDX,  0, 32, VENDOR2,                           YES,  4, TRUE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_1                                               \
FIELD(  1,  0, EAX,  0,  4, STEPPING,                          ANY,  4, FALSE) \
FIELD(  1,  0, EAX,  4,  4, MODEL,                             ANY,  4, FALSE) \
FIELD(  1,  0, EAX,  8,  4, FAMILY,                            YES,  4, FALSE) \
FIELD(  1,  0, EAX, 12,  2, TYPE,                              ANY,  4, FALSE) \
FIELD(  1,  0, EAX, 16,  4, EXTENDED_MODEL,                    ANY,  4, FALSE) \
FIELD(  1,  0, EAX, 20,  8, EXTENDED_FAMILY,                   YES,  4, FALSE) \
FIELD(  1,  0, EBX,  0,  8, BRAND_ID,                          ANY,  4, FALSE) \
FIELD(  1,  0, EBX,  8,  8, CLFL_SIZE,                         ANY,  4, FALSE) \
FIELD(  1,  0, EBX, 16,  8, LCPU_COUNT,                        ANY,  4, FALSE) \
FIELD(  1,  0, EBX, 24,  8, APICID,                            ANY,  4, FALSE) \
FLAG(   1,  0, ECX,  0,  1, SSE3,                              YES,  4, TRUE)  \
FLAG(   1,  0, ECX,  1,  1, PCLMULQDQ,                         YES,  7, TRUE)  \
FLAG(   1,  0, ECX,  2,  1, DTES64,                            NO,   0, FALSE) \
FLAG(   1,  0, ECX,  3,  1, MWAIT,                             YES,  4, FALSE) \
FLAG(   1,  0, ECX,  4,  1, DSCPL,                             NO,   0, FALSE) \
FLAG(   1,  0, ECX,  5,  1, VMX,                               YES,  4, FALSE) \
FLAG(   1,  0, ECX,  6,  1, SMX,                               YES, 15, FALSE) \
FLAG(   1,  0, ECX,  7,  1, EIST,                              NO,   0, FALSE) \
FLAG(   1,  0, ECX,  8,  1, TM2,                               NO,   0, FALSE) \
FLAG(   1,  0, ECX,  9,  1, SSSE3,                             YES,  4, TRUE)  \
FLAG(   1,  0, ECX, 10,  1, CNXTID,                            NO,   0, FALSE) \
FLAG(   1,  0, ECX, 11,  1, SDBG,                              NO,   0, FALSE) \
FLAG(   1,  0, ECX, 12,  1, FMA,                               YES,  8, TRUE)  \
FLAG(   1,  0, ECX, 13,  1, CMPXCHG16B,                        YES,  4, TRUE)  \
FLAG(   1,  0, ECX, 14,  1, xTPR,                              NO,   0, FALSE) \
FLAG(   1,  0, ECX, 15,  1, PDCM,                              NO,   0, FALSE) \
FLAG(   1,  0, ECX, 17,  1, PCID,                              YES,  8, FALSE) \
FLAG(   1,  0, ECX, 18,  1, DCA,                               NO,   0, FALSE) \
FLAG(   1,  0, ECX, 19,  1, SSE41,                             YES,  4, TRUE)  \
FLAG(   1,  0, ECX, 20,  1, SSE42,                             YES,  4, TRUE)  \
FLAG(   1,  0, ECX, 21,  1, x2APIC,                            ANY,  9, FALSE) \
FLAG(   1,  0, ECX, 22,  1, MOVBE,                             YES,  7, TRUE)  \
FLAG(   1,  0, ECX, 23,  1, POPCNT,                            YES,  4, TRUE)  \
FLAG(   1,  0, ECX, 24,  1, TSC_DEADLINE,                      ANY, 11, FALSE) \
FLAG(   1,  0, ECX, 25,  1, AES,                               YES,  7, TRUE)  \
FLAG(   1,  0, ECX, 26,  1, XSAVE,                             YES,  8, FALSE) \
FLAG(   1,  0, ECX, 27,  1, OSXSAVE,                           ANY,  8, FALSE) \
FLAG(   1,  0, ECX, 28,  1, AVX,                               YES,  8, FALSE) \
FLAG(   1,  0, ECX, 29,  1, F16C,                              YES,  9, TRUE)  \
FLAG(   1,  0, ECX, 30,  1, RDRAND,                            YES,  9, TRUE)  \
FLAG(   1,  0, ECX, 31,  1, HYPERVISOR,                        ANY,  4, TRUE)  \
FLAG(   1,  0, EDX,  0,  1, FPU,                               YES,  4, TRUE)  \
FLAG(   1,  0, EDX,  1,  1, VME,                               YES,  4, FALSE) \
FLAG(   1,  0, EDX,  2,  1, DE,                                YES,  4, FALSE) \
FLAG(   1,  0, EDX,  3,  1, PSE,                               YES,  4, FALSE) \
FLAG(   1,  0, EDX,  4,  1, TSC,                               YES,  4, TRUE)  \
FLAG(   1,  0, EDX,  5,  1, MSR,                               YES,  4, FALSE) \
FLAG(   1,  0, EDX,  6,  1, PAE,                               YES,  4, FALSE) \
FLAG(   1,  0, EDX,  7,  1, MCE,                               YES,  4, FALSE) \
FLAG(   1,  0, EDX,  8,  1, CX8,                               YES,  4, TRUE)  \
FLAG(   1,  0, EDX,  9,  1, APIC,                              ANY,  4, FALSE) \
FLAG(   1,  0, EDX, 11,  1, SEP,                               YES,  4, TRUE)  \
FLAG(   1,  0, EDX, 12,  1, MTRR,                              YES,  4, FALSE) \
FLAG(   1,  0, EDX, 13,  1, PGE,                               YES,  4, FALSE) \
FLAG(   1,  0, EDX, 14,  1, MCA,                               YES,  4, FALSE) \
FLAG(   1,  0, EDX, 15,  1, CMOV,                              YES,  4, TRUE)  \
FLAG(   1,  0, EDX, 16,  1, PAT,                               YES,  4, FALSE) \
FLAG(   1,  0, EDX, 17,  1, PSE36,                             YES,  4, FALSE) \
FLAG(   1,  0, EDX, 18,  1, PSN,                               YES,  4, FALSE) \
FLAG(   1,  0, EDX, 19,  1, CLFSH,                             YES,  4, TRUE)  \
FLAG(   1,  0, EDX, 21,  1, DS,                                YES,  4, FALSE) \
FLAG(   1,  0, EDX, 22,  1, ACPI,                              ANY,  4, FALSE) \
FLAG(   1,  0, EDX, 23,  1, MMX,                               YES,  4, TRUE)  \
FLAG(   1,  0, EDX, 24,  1, FXSR,                              YES,  4, TRUE)  \
FLAG(   1,  0, EDX, 25,  1, SSE,                               YES,  4, TRUE)  \
FLAG(   1,  0, EDX, 26,  1, SSE2,                              YES,  4, TRUE)  \
FLAG(   1,  0, EDX, 27,  1, SS,                                YES,  4, FALSE) \
FLAG(   1,  0, EDX, 28,  1, HTT,                               ANY,  7, FALSE) \
FLAG(   1,  0, EDX, 29,  1, TM,                                NO,   0, FALSE) \
FLAG(   1,  0, EDX, 30,  1, IA64,                              NO,   0, FALSE) \
FLAG(   1,  0, EDX, 31,  1, PBE,                               NO,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_2                                               \
FIELD(  2,  0, EAX,  0,  8, LEAF2_COUNT,                       NA,   0, FALSE) \
FIELD(  2,  0, EAX,  8,  8, LEAF2_CACHE1,                      NA,   0, FALSE) \
FIELD(  2,  0, EAX, 16,  8, LEAF2_CACHE2,                      NA,   0, FALSE) \
FIELD(  2,  0, EAX, 24,  8, LEAF2_CACHE3,                      NA,   0, FALSE) \
FIELD(  2,  0, EBX,  0,  8, LEAF2_CACHE4,                      NA,   0, FALSE) \
FIELD(  2,  0, EBX,  8,  8, LEAF2_CACHE5,                      NA,   0, FALSE) \
FIELD(  2,  0, EBX, 16,  8, LEAF2_CACHE6,                      NA,   0, FALSE) \
FIELD(  2,  0, EBX, 24,  8, LEAF2_CACHE7,                      NA,   0, FALSE) \
FIELD(  2,  0, ECX,  0,  8, LEAF2_CACHE8,                      NA,   0, FALSE) \
FIELD(  2,  0, ECX,  8,  8, LEAF2_CACHE9,                      NA,   0, FALSE) \
FIELD(  2,  0, ECX, 16,  8, LEAF2_CACHE10,                     NA,   0, FALSE) \
FIELD(  2,  0, ECX, 24,  8, LEAF2_CACHE11,                     NA,   0, FALSE) \
FIELD(  2,  0, EDX,  0,  8, LEAF2_CACHE12,                     NA,   0, FALSE) \
FIELD(  2,  0, EDX,  8,  8, LEAF2_CACHE13,                     NA,   0, FALSE) \
FIELD(  2,  0, EDX, 16,  8, LEAF2_CACHE14,                     NA,   0, FALSE) \
FIELD(  2,  0, EDX, 24,  8, LEAF2_CACHE15,                     NA,   0, FALSE) \

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_4                                               \
FIELD(  4,  0, EAX,  0,  5, LEAF4_CACHE_TYPE,                  NA,   0, FALSE) \
FIELD(  4,  0, EAX,  5,  3, LEAF4_CACHE_LEVEL,                 NA,   0, FALSE) \
FLAG(   4,  0, EAX,  8,  1, LEAF4_CACHE_SELF_INIT,             NA,   0, FALSE) \
FLAG(   4,  0, EAX,  9,  1, LEAF4_CACHE_FULLY_ASSOC,           NA,   0, FALSE) \
FIELD(  4,  0, EAX, 14, 12, LEAF4_CACHE_NUMHT_SHARING,         NA,   0, FALSE) \
FIELD(  4,  0, EAX, 26,  6, LEAF4_CORE_COUNT,                  NA,   0, FALSE) \
FIELD(  4,  0, EBX,  0, 12, LEAF4_CACHE_LINE,                  NA,   0, FALSE) \
FIELD(  4,  0, EBX, 12, 10, LEAF4_CACHE_PART,                  NA,   0, FALSE) \
FIELD(  4,  0, EBX, 22, 10, LEAF4_CACHE_WAYS,                  NA,   0, FALSE) \
FIELD(  4,  0, ECX,  0, 32, LEAF4_CACHE_SETS,                  NA,   0, FALSE) \
FLAG(   4,  0, EDX,  0,  1, LEAF4_CACHE_WBINVD_NOT_GUARANTEED, NA,   0, FALSE) \
FLAG(   4,  0, EDX,  1,  1, LEAF4_CACHE_IS_INCLUSIVE,          NA,   0, FALSE) \
FLAG(   4,  0, EDX,  2,  1, LEAF4_CACHE_COMPLEX_INDEXING,      NA,   0, FALSE)

/*     LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,            MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_5                                               \
FIELD(  5,  0, EAX,  0, 16, MWAIT_MIN_SIZE,                    NA,   0, FALSE) \
FIELD(  5,  0, EBX,  0, 16, MWAIT_MAX_SIZE,                    NA,   0, FALSE) \
FLAG(   5,  0, ECX,  0,  1, MWAIT_EXTENSIONS,                  NA,   0, FALSE) \
FLAG(   5,  0, ECX,  1,  1, MWAIT_INTR_BREAK,                  NA,   0, FALSE) \
FIELD(  5,  0, EDX,  0,  4, MWAIT_C0_SUBSTATE,                 NA,   0, FALSE) \
FIELD(  5,  0, EDX,  4,  4, MWAIT_C1_SUBSTATE,                 NA,   0, FALSE) \
FIELD(  5,  0, EDX,  8,  4, MWAIT_C2_SUBSTATE,                 NA,   0, FALSE) \
FIELD(  5,  0, EDX, 12,  4, MWAIT_C3_SUBSTATE,                 NA,   0, FALSE) \
FIELD(  5,  0, EDX, 16,  4, MWAIT_C4_SUBSTATE,                 NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_6                                               \
FLAG(   6,  0, EAX,  0,  1, THERMAL_SENSOR,                    NO,   0, FALSE) \
FLAG(   6,  0, EAX,  1,  1, TURBO_MODE,                        NO,   0, FALSE) \
FLAG(   6,  0, EAX,  2,  1, APIC_INVARIANT,                    ANY, 13, FALSE) \
FLAG(   6,  0, EAX,  4,  1, PLN,                               NO,   0, FALSE) \
FLAG(   6,  0, EAX,  5,  1, ECMD,                              NO,   0, FALSE) \
FLAG(   6,  0, EAX,  6,  1, PTM,                               NO,   0, FALSE) \
FLAG(   6,  0, EAX,  7,  1, HWP,                               NO,   0, FALSE) \
FLAG(   6,  0, EAX,  8,  1, HWP_NOTIFICATION,                  NO,   0, FALSE) \
FLAG(   6,  0, EAX,  9,  1, HWP_ACTIVITY_WINDOW,               NO,   0, FALSE) \
FLAG(   6,  0, EAX, 10,  1, HWP_ENERGY_PERFORMANCE_PREFERENCE, NO,   0, FALSE) \
FLAG(   6,  0, EAX, 11,  1, HWP_PACKAGE_LEVEL_REQUEST,         NO,   0, FALSE) \
FLAG(   6,  0, EAX, 13,  1, HDC,                               NO,   0, FALSE) \
FIELD(  6,  0, EBX,  0,  4, NUM_INTR_THRESHOLDS,               NO,   0, FALSE) \
FLAG(   6,  0, ECX,  0,  1, HW_COORD_FEEDBACK,                 NO,   0, FALSE) \
FLAG(   6,  0, ECX,  1,  1, ACNT2,                             ANY, 13, FALSE) \
FLAG(   6,  0, ECX,  3,  1, ENERGY_PERF_BIAS,                  NO,   0, FALSE)

#define CPUID_7_EDX_28 \
FLAG(   7,  0, EDX, 28,  1, LEVEL7EDX_RSVD1,                    NO,  0, FALSE)

#define CPUID_7_EDX_31 \
FLAG(   7,  0, EDX, 31,  1, LEVEL7EDX_RSVD2,                    NO,  0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_7                                               \
FLAG(   7,  0, EBX,  0,  1, FSGSBASE,                          YES,  9, FALSE) \
FLAG(   7,  0, EBX,  1,  1, TSC_ADJUST,                        ANY, 11, FALSE) \
FLAG(   7,  0, EBX,  2,  1, SGX,                               YES, 16, FALSE) \
FLAG(   7,  0, EBX,  3,  1, BMI1,                              YES,  9, TRUE)  \
FLAG(   7,  0, EBX,  4,  1, HLE,                               YES, 11, TRUE)  \
FLAG(   7,  0, EBX,  5,  1, AVX2,                              YES, 11, TRUE)  \
FLAG(   7,  0, EBX,  6,  1, FDP_EXCPTN_ONLY,                   ANY, 13, TRUE)  \
FLAG(   7,  0, EBX,  7,  1, SMEP,                              YES,  9, FALSE) \
FLAG(   7,  0, EBX,  8,  1, BMI2,                              YES, 11, TRUE)  \
FLAG(   7,  0, EBX,  9,  1, ENFSTRG,                           YES,  9, FALSE) \
FLAG(   7,  0, EBX, 10,  1, INVPCID,                           YES, 11, FALSE) \
FLAG(   7,  0, EBX, 11,  1, RTM,                               YES, 11, TRUE)  \
FLAG(   7,  0, EBX, 12,  1, PQM,                               NO,   0, FALSE) \
FLAG(   7,  0, EBX, 13,  1, FP_SEGMENT_ZERO,                   ANY, 11, TRUE)  \
FLAG(   7,  0, EBX, 14,  1, MPX,                               YES, 13, TRUE)  \
FLAG(   7,  0, EBX, 15,  1, PQE,                               NO,   0, FALSE) \
FLAG(   7,  0, EBX, 16,  1, AVX512F,                           YES, 13, TRUE)  \
FLAG(   7,  0, EBX, 17,  1, AVX512DQ,                          YES, 13, TRUE)  \
FLAG(   7,  0, EBX, 18,  1, RDSEED,                            YES, 11, TRUE)  \
FLAG(   7,  0, EBX, 19,  1, ADX,                               YES, 11, TRUE)  \
FLAG(   7,  0, EBX, 20,  1, SMAP,                              YES, 11, FALSE) \
FLAG(   7,  0, EBX, 21,  1, AVX512IFMA,                        YES, 16, TRUE)  \
FLAG(   7,  0, EBX, 23,  1, CLFLUSHOPT,                        YES, 13, TRUE)  \
FLAG(   7,  0, EBX, 24,  1, CLWB,                              YES, 13, TRUE)  \
FLAG(   7,  0, EBX, 25,  1, PT,                                YES, 16, FALSE) \
FLAG(   7,  0, EBX, 26,  1, AVX512PF,                          YES, 13, TRUE)  \
FLAG(   7,  0, EBX, 27,  1, AVX512ER,                          YES, 13, TRUE)  \
FLAG(   7,  0, EBX, 28,  1, AVX512CD,                          YES, 13, TRUE)  \
FLAG(   7,  0, EBX, 29,  1, SHA,                               YES, 14, TRUE)  \
FLAG(   7,  0, EBX, 30,  1, AVX512BW,                          YES, 13, TRUE)  \
FLAG(   7,  0, EBX, 31,  1, AVX512VL,                          YES, 13, TRUE)  \
FLAG(   7,  0, ECX,  0,  1, PREFETCHWT1,                       YES, 13, TRUE)  \
FLAG(   7,  0, ECX,  1,  1, AVX512VBMI,                        YES, 16, TRUE)  \
FLAG(   7,  0, ECX,  2,  1, UMIP,                              NO,   0, FALSE) \
FLAG(   7,  0, ECX,  3,  1, PKU,                               YES, 13, TRUE)  \
FLAG(   7,  0, ECX,  4,  1, OSPKE,                             ANY, 13, TRUE)  \
FLAG(   7,  0, ECX,  6,  1, AVX512VBMI2,                       NO,   0, TRUE)  \
FLAG(   7,  0, ECX,  8,  1, GFNI,                              NO,   0, TRUE)  \
FLAG(   7,  0, ECX,  9,  1, VAES,                              NO,   0, TRUE)  \
FLAG(   7,  0, ECX, 10,  1, VPCLMULQDQ,                        NO,   0, TRUE)  \
FLAG(   7,  0, ECX, 11,  1, AVX512VNNI,                        NO,   0, TRUE)  \
FLAG(   7,  0, ECX, 12,  1, AVX512BITALG,                      NO,   0, TRUE)  \
FLAG(   7,  0, ECX, 14,  1, AVX512VPOPCNTDQ,                   YES, 15, TRUE)  \
FLAG(   7,  0, ECX, 16,  1, VA57,                              NO,   0, TRUE)  \
FIELD(  7,  0, ECX, 17,  5, MAWA,                              NO,   0, TRUE)  \
FLAG(   7,  0, ECX, 22,  1, RDPID,                             NO,   0, TRUE)  \
FLAG(   7,  0, ECX, 30,  1, SGX_LC,                            YES, 16, FALSE) \
FLAG(   7,  0, EDX,  2,  1, AVX512QVNNIW,                      YES, 15, TRUE)  \
FLAG(   7,  0, EDX,  3,  1, AVX512QFMAPS,                      YES, 15, TRUE)  \
FLAG(   7,  0, EDX, 26,  1, IBRSIBPB,                          ANY,  9, FALSE) \
FLAG(   7,  0, EDX, 27,  1, STIBP,                             YES,  9, FALSE) \
CPUID_7_EDX_28                                                                 \
FLAG(   7,  0, EDX, 29,  1, ARCH_CAPABILITIES,                 ANY,  9, FALSE) \
CPUID_7_EDX_31

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_A                                               \
FIELD(  A,  0, EAX,  0,  8, PMC_VERSION,                       NA,   0, FALSE) \
FIELD(  A,  0, EAX,  8,  8, PMC_NUM_GEN,                       NA,   0, FALSE) \
FIELD(  A,  0, EAX, 16,  8, PMC_WIDTH_GEN,                     NA,   0, FALSE) \
FIELD(  A,  0, EAX, 24,  8, PMC_EBX_LENGTH,                    NA,   0, FALSE) \
FLAG(   A,  0, EBX,  0,  1, PMC_CORE_CYCLES,                   NA,   0, FALSE) \
FLAG(   A,  0, EBX,  1,  1, PMC_INSTR_RETIRED,                 NA,   0, FALSE) \
FLAG(   A,  0, EBX,  2,  1, PMC_REF_CYCLES,                    NA,   0, FALSE) \
FLAG(   A,  0, EBX,  3,  1, PMC_LAST_LVL_CREF,                 NA,   0, FALSE) \
FLAG(   A,  0, EBX,  4,  1, PMC_LAST_LVL_CMISS,                NA,   0, FALSE) \
FLAG(   A,  0, EBX,  5,  1, PMC_BR_INST_RETIRED,               NA,   0, FALSE) \
FLAG(   A,  0, EBX,  6,  1, PMC_BR_MISS_RETIRED,               NA,   0, FALSE) \
FIELD(  A,  0, EDX,  0,  5, PMC_NUM_FIXED,                     NA,   0, FALSE) \
FIELD(  A,  0, EDX,  5,  8, PMC_WIDTH_FIXED,                   NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_B                                               \
FIELD(  B,  0, EAX,  0,  5, TOPOLOGY_MASK_WIDTH,               NA,   0, FALSE) \
FIELD(  B,  0, EBX,  0, 16, TOPOLOGY_CPUS_SHARING_LEVEL,       NA,   0, FALSE) \
FIELD(  B,  0, ECX,  0,  8, TOPOLOGY_LEVEL_NUMBER,             NA,   0, FALSE) \
FIELD(  B,  0, ECX,  8,  8, TOPOLOGY_LEVEL_TYPE,               NA,   0, FALSE) \
FIELD(  B,  0, EDX,  0, 32, TOPOLOGY_X2APIC_ID,                NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_D                                               \
FLAG(   D,  0, EAX,  0,  1, XCR0_MASTER_LEGACY_FP,             YES,  8, FALSE) \
FLAG(   D,  0, EAX,  1,  1, XCR0_MASTER_SSE,                   YES,  8, FALSE) \
FLAG(   D,  0, EAX,  2,  1, XCR0_MASTER_YMM_H,                 YES,  8, FALSE) \
FLAG(   D,  0, EAX,  3,  1, XCR0_MASTER_BNDREGS,               YES, 13, FALSE) \
FLAG(   D,  0, EAX,  4,  1, XCR0_MASTER_BNDCSR,                YES, 13, FALSE) \
FLAG(   D,  0, EAX,  5,  1, XCR0_MASTER_OPMASK,                YES, 13, FALSE) \
FLAG(   D,  0, EAX,  6,  1, XCR0_MASTER_ZMM_H,                 YES, 13, FALSE) \
FLAG(   D,  0, EAX,  7,  1, XCR0_MASTER_HI16_ZMM,              YES, 13, FALSE) \
FLAG(   D,  0, EAX,  8,  1, XCR0_MASTER_XSS,                   NO,   0, FALSE) \
FLAG(   D,  0, EAX,  9,  1, XCR0_MASTER_PKRU,                  YES, 13, FALSE) \
FIELD(  D,  0, EAX,  10,22, XCR0_MASTER_LOWER,                 NO,   0, FALSE) \
FIELD(  D,  0, EBX,  0, 32, XSAVE_ENABLED_SIZE,                ANY,  8, FALSE) \
FIELD(  D,  0, ECX,  0, 32, XSAVE_MAX_SIZE,                    YES,  8, FALSE) \
FIELD(  D,  0, EDX,  0, 29, XCR0_MASTER_UPPER,                 NO,   0, FALSE) \
FLAG(   D,  0, EDX, 30,  1, XCR0_MASTER_LWP,                   NO,   0, FALSE) \
FLAG(   D,  0, EDX, 31,  1, XCR0_MASTER_EXTENDED_XSAVE,        NO,   0, FALSE) \
FLAG(   D,  1, EAX,  0,  1, XSAVEOPT,                          YES, 11, FALSE) \
FLAG(   D,  1, EAX,  1,  1, XSAVEC,                            YES, 13, FALSE) \
FLAG(   D,  1, EAX,  2,  1, XGETBV_ECX1,                       NO,   0, FALSE) \
FLAG(   D,  1, EAX,  3,  1, XSAVES,                            YES, 13, FALSE) \
FIELD(  D,  1, EBX,  0, 32, XSAVES_ENABLED_SIZE,               ANY, 13, FALSE) \
FIELD(  D,  1, ECX,  0,  7, XSS_XCR0_USED0,                    NO,   0, FALSE) \
FLAG(   D,  1, ECX,  8,  1, XSS_PT,                            NO,   0, FALSE) \
FIELD(  D,  1, ECX,  9,  1, XSS_XCR0_USED1,                    NO,   0, FALSE) \
FIELD(  D,  1, ECX,  10,22, XSS_RSVD0,                         NO,   0, FALSE) \
FIELD(  D,  1, EDX,  0, 32, XSS_RSVD1,                         NO,   0, FALSE) \
FIELD(  D,  2, EAX,  0, 32, XSAVE_YMM_SIZE,                    YES,  9, FALSE) \
FIELD(  D,  2, EBX,  0, 32, XSAVE_YMM_OFFSET,                  YES,  9, FALSE) \
FLAG(   D,  2, ECX,  0,  1, XSAVE_YMM_SUP_BY_XSS,              NO,   0, FALSE) \
FLAG(   D,  2, ECX,  1,  1, XSAVE_YMM_ALIGN,                   YES, 13, FALSE) \
FIELD(  D,  2, ECX,  2, 30, XSAVE_YMM_RSVD1,                   NO,   0, FALSE) \
FIELD(  D,  2, EDX,  0, 32, XSAVE_YMM_RSVD2,                   NO,   0, FALSE) \
FIELD(  D,  3, EAX,  0, 32, XSAVE_BNDREGS_SIZE,                YES, 13, FALSE) \
FIELD(  D,  3, EBX,  0, 32, XSAVE_BNDREGS_OFFSET,              YES, 13, FALSE) \
FLAG(   D,  3, ECX,  0,  1, XSAVE_BNDREGS_SUP_BY_XSS,          NO,   0, FALSE) \
FLAG(   D,  3, ECX,  1,  1, XSAVE_BNDREGS_ALIGN,               YES, 13, FALSE) \
FIELD(  D,  3, ECX,  2, 30, XSAVE_BNDREGS_RSVD1,               NO,   0, FALSE) \
FIELD(  D,  3, EDX,  0, 32, XSAVE_BNDREGS_RSVD2,               NO,   0, FALSE) \
FIELD(  D,  4, EAX,  0, 32, XSAVE_BNDCSR_SIZE,                 YES, 13, FALSE) \
FIELD(  D,  4, EBX,  0, 32, XSAVE_BNDCSR_OFFSET,               YES, 13, FALSE) \
FLAG(   D,  4, ECX,  0,  1, XSAVE_BNDCSR_SUP_BY_XSS,           NO,   0, FALSE) \
FLAG(   D,  4, ECX,  1,  1, XSAVE_BNDCSR_ALIGN,                YES, 13, FALSE) \
FIELD(  D,  4, ECX,  2, 30, XSAVE_BNDCSR_RSVD1,                NO,   0, FALSE) \
FIELD(  D,  4, EDX,  0, 32, XSAVE_BNDCSR_RSVD2,                NO,   0, FALSE) \
FIELD(  D,  5, EAX,  0, 32, XSAVE_OPMASK_SIZE,                 YES, 13, FALSE) \
FIELD(  D,  5, EBX,  0, 32, XSAVE_OPMASK_OFFSET,               YES, 13, FALSE) \
FLAG(   D,  5, ECX,  0,  1, XSAVE_OPMASK_SUP_BY_XSS,           NO,   0, FALSE) \
FLAG(   D,  5, ECX,  1,  1, XSAVE_OPMASK_ALIGN,                YES, 13, FALSE) \
FIELD(  D,  5, ECX,  2, 30, XSAVE_OPMASK_RSVD1,                NO,   0, FALSE) \
FIELD(  D,  5, EDX,  0, 32, XSAVE_OPMASK_RSVD2,                NO,   0, FALSE) \
FIELD(  D,  6, EAX,  0, 32, XSAVE_ZMM_H_SIZE,                  YES, 13, FALSE) \
FIELD(  D,  6, EBX,  0, 32, XSAVE_ZMM_H_OFFSET,                YES, 13, FALSE) \
FLAG(   D,  6, ECX,  0,  1, XSAVE_ZMM_H_SUP_BY_XSS,            NO,   0, FALSE) \
FLAG(   D,  6, ECX,  1,  1, XSAVE_ZMM_H_ALIGN,                 YES, 13, FALSE) \
FIELD(  D,  6, ECX,  2, 30, XSAVE_ZMM_H_RSVD1,                 NO,   0, FALSE) \
FIELD(  D,  6, EDX,  0, 32, XSAVE_ZMM_H_RSVD2,                 NO,   0, FALSE) \
FIELD(  D,  7, EAX,  0, 32, XSAVE_HI16_ZMM_SIZE,               YES, 13, FALSE) \
FIELD(  D,  7, EBX,  0, 32, XSAVE_HI16_ZMM_OFFSET,             YES, 13, FALSE) \
FLAG(   D,  7, ECX,  0,  1, XSAVE_HI16_ZMM_SUP_BY_XSS,         NO,   0, FALSE) \
FLAG(   D,  7, ECX,  1,  1, XSAVE_HI16_ZMM_ALIGN,              YES, 13, FALSE) \
FIELD(  D,  7, ECX,  2, 30, XSAVE_HI16_ZMM_RSVD1,              NO,   0, FALSE) \
FIELD(  D,  7, EDX,  0, 32, XSAVE_HI16_ZMM_RSVD2,              NO,   0, FALSE) \
FIELD(  D,  8, EAX,  0, 32, XSAVE_PT_STATE_SIZE,               NO,   0, FALSE) \
FIELD(  D,  8, EBX,  0, 32, XSAVE_PT_STATE_OFFSET,             NO,   0, FALSE) \
FLAG(   D,  8, ECX,  0,  1, XSAVE_PT_STATE_SUP_BY_XSS,         NO,   0, FALSE) \
FLAG(   D,  8, ECX,  1,  1, XSAVE_PT_STATE_ALIGN,              NO,   0, FALSE) \
FIELD(  D,  8, ECX,  2, 30, XSAVE_PT_STATE_RSVD1,              NO,   0, FALSE) \
FIELD(  D,  8, EDX,  0, 32, XSAVE_PT_STATE_RSVD2,              NO,   0, FALSE) \
FIELD(  D,  9, EAX,  0, 32, XSAVE_PKRU_SIZE,                   YES, 13, FALSE) \
FIELD(  D,  9, EBX,  0, 32, XSAVE_PKRU_OFFSET,                 YES, 13, FALSE) \
FLAG(   D,  9, ECX,  0,  1, XSAVE_PKRU_SUP_BY_XSS,             NO,   0, FALSE) \
FLAG(   D,  9, ECX,  1,  1, XSAVE_PKRU_ALIGN,                  YES, 13, FALSE) \
FIELD(  D,  9, ECX,  2, 30, XSAVE_PKRU_RSVD1,                  NO,   0, FALSE) \
FIELD(  D,  9, EDX,  0, 32, XSAVE_PKRU_RSVD2,                  NO,   0, FALSE) \
FIELD(  D, 62, EAX,  0, 32, XSAVE_LWP_SIZE,                    NO,   0, FALSE) \
FIELD(  D, 62, EBX,  0, 32, XSAVE_LWP_OFFSET,                  NO,   0, FALSE) \
FIELD(  D, 62, ECX,  0, 32, XSAVE_LWP_RSVD1,                   NO,   0, FALSE) \
FIELD(  D, 62, EDX,  0, 32, XSAVE_LWP_RSVD2,                   NO,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_F                                               \
FIELD(  F,  0, EBX,  0, 32, PQM_MAX_RMID,                      NO,   0, FALSE) \
FLAG(   F,  0, EDX,  1,  1, PQM_CMT_SUPPORT,                   NO,   0, FALSE) \
FIELD(  F,  1, EBX,  0, 32, PQM_CMT_CONV,                      NO,   0, FALSE) \
FIELD(  F,  1, ECX,  0, 32, PQM_CMT_NUM_RMID,                  NO,   0, FALSE) \
FLAG(   F,  1, EDX,  0,  1, PQM_CMT_OCCUPANCY,                 NO,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_10                                              \
FLAG(  10,  0, EBX,  1,  1, L3_QOS_ENFORCEMENT,                NO,   0, FALSE) \
FIELD( 10,  1, EAX,  0,  4, RESID_CAPACITY_MASK_LENGTH,        NO,   0, FALSE) \
FIELD( 10,  1, EBX,  0, 32, ISOLATION_UNIT_MAP,                NO,   0, FALSE) \
FLAG(  10,  1, ECX,  1,  1, INFREQUENT_COS_UPDATE,             NO,   0, FALSE) \
FLAG(  10,  1, ECX,  2,  1, CODE_AND_DATA_PRIORITIZATION,      NO,   0, FALSE) \
FIELD( 10,  1, EDX,  0, 16, MAX_COS_NUMBER,                    NO,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_12                                              \
FLAG(  12,  0, EAX,  0,  1, SGX1,                              YES, 16, FALSE) \
FLAG(  12,  0, EAX,  1,  1, SGX2,                              NO,   0, FALSE) \
FLAG(  12,  0, EBX,  0,  1, SGX_MISCSELECT_EXINFO,             YES, 16, FALSE) \
FIELD( 12,  0, EBX,  1, 31, SGX_MISCSELECT_RSVD,               NO,   0, FALSE) \
FIELD( 12,  0, EDX,  0,  8, MAX_ENCLAVE_SIZE_NOT64,            YES, 16, FALSE) \
FIELD( 12,  0, EDX,  8,  8, MAX_ENCLAVE_SIZE_64,               YES, 16, FALSE) \
FIELD( 12,  1, EAX,  0, 32, SECS_ATTRIBUTES0,                  YES, 16, FALSE) \
FIELD( 12,  1, EBX,  0, 32, SECS_ATTRIBUTES1,                  YES, 16, FALSE) \
FIELD( 12,  1, ECX,  0, 32, SECS_ATTRIBUTES2,                  YES, 16, FALSE) \
FIELD( 12,  1, EDX,  0, 32, SECS_ATTRIBUTES3,                  YES, 16, FALSE) \
FIELD( 12,  2, EAX,  0, 15, EPC00_VALID,                       YES, 16, FALSE) \
FIELD( 12,  2, EAX, 12, 20, EPC00_BASE_LOW,                    YES, 16, FALSE) \
FIELD( 12,  2, EBX,  0, 20, EPC00_BASE_HIGH,                   YES, 16, FALSE) \
FIELD( 12,  2, ECX,  0, 15, EPC00_PROTECTED,                   YES, 16, FALSE) \
FIELD( 12,  2, ECX, 12, 20, EPC00_SIZE_LOW,                    YES, 16, FALSE) \
FIELD( 12,  2, EDX,  0, 20, EPC00_SIZE_HIGH,                   YES, 16, FALSE) \
FIELD( 12,  3, EAX,  0,  4, EPC01_VALID,                       NO,   0, FALSE) \
FIELD( 12,  3, EAX, 12, 20, EPC01_BASE_LOW,                    NO,   0, FALSE) \
FIELD( 12,  3, EBX,  0, 20, EPC01_BASE_HIGH,                   NO,   0, FALSE) \
FIELD( 12,  3, ECX,  0,  4, EPC01_PROTECTED,                   NO,   0, FALSE) \
FIELD( 12,  3, ECX, 12, 20, EPC01_SIZE_LOW,                    NO,   0, FALSE) \
FIELD( 12,  3, EDX,  0, 20, EPC01_SIZE_HIGH,                   NO,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_14                                              \
FIELD( 14,  0, EAX,  0, 32, MAX_PT_SUB_LEAF,                   YES, 16, FALSE) \
FLAG(  14,  0, EBX,  0,  1, CR3FTR_AND_MATCHMSR_AVAILABLE,     YES, 16, FALSE) \
FLAG(  14,  0, EBX,  1,  1, PSB_AND_CYCLE_ACCURATE_MODE,       YES, 16, FALSE) \
FLAG(  14,  0, EBX,  2,  1, IP_TRACESTOP_FTR_PTMSR_PERSIST,    YES, 16, FALSE) \
FLAG(  14,  0, EBX,  3,  1, MTC_PKT_GENERATION_SUPPORTED,      YES, 16, FALSE) \
FLAG(  14,  0, ECX,  0,  1, TOPA_OUTPUT_SUPPORTED,             YES, 16, FALSE) \
FLAG(  14,  0, ECX,  1,  1, TOPA_ALLOW_MULTIPLE_ENTRIES,       YES, 16, FALSE) \
FLAG(  14,  0, ECX,  2,  1, SINGLE_RANGE_OUTPUT_SCHEME,        YES, 16, FALSE) \
FLAG(  14,  0, ECX,  3,  1, TRACE_TRANSPORT_SUBSYSTEM,         NO,   0, FALSE) \
FLAG(  14,  0, ECX, 31,  1, LIP_PRESENT_FOR_IP_PAYLOADS,       YES, 16, FALSE) \
FIELD( 14,  1, EAX,  0,  2, NUM_ADDR_RANGE_FOR_FILTERING,      YES, 16, FALSE) \
FIELD( 14,  1, EAX, 16, 16, SUPPORTED_MTC_ENCODINGS,           YES, 16, FALSE) \
FIELD( 14,  1, EBX,  0, 16, SUPPORTED_CYCLE_THRESHOLD_ENCODINGS, YES,16,FALSE) \
FIELD( 14,  1, EBX, 16, 16, SUPPORTED_PSB_FREQ_ENCODINGS,      YES, 16, FALSE) \

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_15                                              \
FIELD( 15,  0, EAX,  0, 32, DENOM_TSC_TO_CORE_CRYSTAL_CLK,     NO,   0, FALSE) \
FIELD( 15,  0, EBX,  0, 32, NUMER_TSC_TO_CORE_CRYSTAL_CLK,     NO,   0, FALSE) \

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_16                                              \
FIELD( 16,  0, EAX,  0, 16, PROC_BASE_FREQ,                    NO,   0, FALSE) \
FIELD( 16,  0, EBX,  0, 16, PROC_MIN_FREQ,                     NO,   0, FALSE) \
FIELD( 16,  0, ECX,  0, 16, BUS_FREQ,                          NO,   0, FALSE) \

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_17                                              \
FIELD( 17,  0, EAX,  0, 31, MAX_SOCID_INDEX,                   NO,   0, FALSE) \
FIELD( 17,  0, EBX,  0, 16, SOC_VENDOR_ID,                     NO,   0, FALSE) \
FIELD( 17,  0, EBX, 16,  1, SOC_INDUSTRY_STD,                  NO,   0, FALSE) \
FIELD( 17,  0, ECX,  0, 31, SOC_PROJECT_ID,                    NO,   0, FALSE) \
FIELD( 17,  0, EDX,  0, 31, SOC_STEPPING_ID,                   NO,   0, FALSE) \
FIELD( 17,  1, EAX,  0, 32, SOC_VENDOR_BRAND_STRING_1_0,       NO,   0, FALSE) \
FIELD( 17,  1, EBX,  0, 32, SOC_VENDOR_BRAND_STRING_1_1,       NO,   0, FALSE) \
FIELD( 17,  1, ECX,  0, 32, SOC_VENDOR_BRAND_STRING_1_2,       NO,   0, FALSE) \
FIELD( 17,  1, EDX,  0, 32, SOC_VENDOR_BRAND_STRING_1_3,       NO,   0, FALSE) \
FIELD( 17,  2, EAX,  0, 32, SOC_VENDOR_BRAND_STRING_2_0,       NO,   0, FALSE) \
FIELD( 17,  2, EBX,  0, 32, SOC_VENDOR_BRAND_STRING_2_1,       NO,   0, FALSE) \
FIELD( 17,  2, ECX,  0, 32, SOC_VENDOR_BRAND_STRING_2_2,       NO,   0, FALSE) \
FIELD( 17,  2, EDX,  0, 32, SOC_VENDOR_BRAND_STRING_2_3,       NO,   0, FALSE) \
FIELD( 17,  3, EAX,  0, 32, SOC_VENDOR_BRAND_STRING_3_0,       NO,   0, FALSE) \
FIELD( 17,  3, EBX,  0, 32, SOC_VENDOR_BRAND_STRING_3_1,       NO,   0, FALSE) \
FIELD( 17,  3, ECX,  0, 32, SOC_VENDOR_BRAND_STRING_3_2,       NO,   0, FALSE) \
FIELD( 17,  3, EDX,  0, 32, SOC_VENDOR_BRAND_STRING_3_3,       NO,   0, FALSE) \

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_400                                             \
FIELD(400,  0, EAX,  0, 32, MAX_HYP_LEVEL,                     NA,   0, FALSE) \
FIELD(400,  0, EBX,  0, 32, HYPERVISOR_VENDOR0,                NA,   0, FALSE) \
FIELD(400,  0, ECX,  0, 32, HYPERVISOR_VENDOR1,                NA,   0, FALSE) \
FIELD(400,  0, EDX,  0, 32, HYPERVISOR_VENDOR2,                NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_401                                             \
FIELD(401,  0, EAX,  0, 32, HV_INTERFACE_SIGNATURE,            NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_402                                             \
FIELD(402,  0, EAX,  0, 32, BUILD_NUMBER,                      NA,   0, FALSE) \
FIELD(402,  0, EBX,  0, 16, MINOR_VERSION,                     NA,   0, FALSE) \
FIELD(402,  0, EBX, 16, 16, MAJOR_VERSION,                     NA,   0, FALSE) \
FIELD(402,  0, ECX,  0, 32, SERVICE_PACK,                      NA,   0, FALSE) \
FIELD(402,  0, EDX,  0, 24, SERVICE_NUMBER,                    NA,   0, FALSE) \
FIELD(402,  0, EDX, 24,  8, SERVICE_BRANCH,                    NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_403                                             \
FLAG( 403,  0, EAX,  0,  1, VP_RUNTIME_AVAIL,                  NA,   0, FALSE) \
FLAG( 403,  0, EAX,  1,  1, REF_COUNTER_AVAIL,                 NA,   0, FALSE) \
FLAG( 403,  0, EAX,  2,  1, BASIC_SYNIC_MSRS_AVAIL,            NA,   0, FALSE) \
FLAG( 403,  0, EAX,  3,  1, SYNTH_TIMER_MSRS_AVAIL,            NA,   0, FALSE) \
FLAG( 403,  0, EAX,  4,  1, APIC_ACCESS_MSRS_AVAIL,            NA,   0, FALSE) \
FLAG( 403,  0, EAX,  5,  1, HYPERCALL_MSRS_AVAIL,              NA,   0, FALSE) \
FLAG( 403,  0, EAX,  6,  1, VP_INDEX_MSR_AVAIL,                NA,   0, FALSE) \
FLAG( 403,  0, EAX,  7,  1, VIRT_RESET_MSR_AVAIL,              NA,   0, FALSE) \
FLAG( 403,  0, EAX,  8,  1, STATS_PAGES_MSRS_AVAIL,            NA,   0, FALSE) \
FLAG( 403,  0, EAX,  9,  1, REF_TSC_AVAIL,                     NA,   0, FALSE) \
FLAG( 403,  0, EAX, 10,  1, GUEST_IDLE_MSR_AVAIL,              NA,   0, FALSE) \
FLAG( 403,  0, EAX, 11,  1, FREQUENCY_MSRS_AVAIL,              NA,   0, FALSE) \
FLAG( 403,  0, EAX, 12,  1, SYNTH_DEBUG_MSRS_AVAIL,            NA,   0, FALSE) \
FLAG( 403,  0, EBX,  0,  1, CREATE_PARTITIONS_FLAG,            NA,   0, FALSE) \
FLAG( 403,  0, EBX,  1,  1, ACCESS_PARTITION_ID_FLAG,          NA,   0, FALSE) \
FLAG( 403,  0, EBX,  2,  1, ACCESS_MEMORY_POOL_FLAG,           NA,   0, FALSE) \
FLAG( 403,  0, EBX,  3,  1, ADJUST_MESSAGE_BUFFERS_FLAG,       NA,   0, FALSE) \
FLAG( 403,  0, EBX,  4,  1, POST_MESSAGES_FLAG,                NA,   0, FALSE) \
FLAG( 403,  0, EBX,  5,  1, SIGNAL_EVENTS_FLAG,                NA,   0, FALSE) \
FLAG( 403,  0, EBX,  6,  1, CREATE_PORT_FLAG,                  NA,   0, FALSE) \
FLAG( 403,  0, EBX,  7,  1, CONNECT_PORT_FLAG,                 NA,   0, FALSE) \
FLAG( 403,  0, EBX,  8,  1, ACCESS_STATS_FLAG,                 NA,   0, FALSE) \
FLAG( 403,  0, EBX, 11,  1, DEBUGGING_FLAG,                    NA,   0, FALSE) \
FLAG( 403,  0, EBX, 12,  1, CPU_MANAGEMENT_FLAG,               NA,   0, FALSE) \
FLAG( 403,  0, EBX, 13,  1, CONFIGURE_PROFILER_FLAG,           NA,   0, FALSE) \
FLAG( 403,  0, EBX, 14,  1, ENABLE_EXPANDED_STACKWALKING_FLAG, NA,   0, FALSE) \
FIELD(403,  0, ECX,  0,  4, MAX_POWER_STATE,                   NA,   0, FALSE) \
FLAG( 403,  0, ECX,  4,  1, HPET_NEEDED_FOR_C3,                NA,   0, FALSE) \
FLAG( 403,  0, EDX,  0,  1, MWAIT_AVAIL,                       NA,   0, FALSE) \
FLAG( 403,  0, EDX,  1,  1, GUEST_DEBUGGING_AVAIL,             NA,   0, FALSE) \
FLAG( 403,  0, EDX,  2,  1, PERFORMANCE_MONITOR_AVAIL,         NA,   0, FALSE) \
FLAG( 403,  0, EDX,  3,  1, CPU_DYN_PARTITIONING_AVAIL,        NA,   0, FALSE) \
FLAG( 403,  0, EDX,  4,  1, XMM_REGISTERS_FOR_HYPERCALL_AVAIL, NA,   0, FALSE) \
FLAG( 403,  0, EDX,  5,  1, GUEST_IDLE_AVAIL,                  NA,   0, FALSE) \
FLAG( 403,  0, EDX,  6,  1, HYPERVISOR_SLEEP_STATE_AVAIL,      NA,   0, FALSE) \
FLAG( 403,  0, EDX,  7,  1, NUMA_DISTANCE_QUERY_AVAIL,         NA,   0, FALSE) \
FLAG( 403,  0, EDX,  8,  1, TIMER_FREQUENCY_AVAIL,             NA,   0, FALSE) \
FLAG( 403,  0, EDX,  9,  1, SYNTH_MACHINE_CHECK_AVAIL,         NA,   0, FALSE) \
FLAG( 403,  0, EDX, 10,  1, GUEST_CRASH_MSRS_AVAIL,            NA,   0, FALSE) \
FLAG( 403,  0, EDX, 11,  1, DEBUG_MSRS_AVAIL,                  NA,   0, FALSE) \
FLAG( 403,  0, EDX, 12,  1, NPIEP1_AVAIL,                      NA,   0, FALSE) \
FLAG( 403,  0, EDX, 13,  1, DISABLE_HYPERVISOR_AVAIL,          NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_404                                             \
FLAG( 404,  0, EAX,  0,  1, USE_HYPERCALL_TO_SWITCH_ADDR_SPACE,NA,   0, FALSE) \
FLAG( 404,  0, EAX,  1,  1, USE_HYPERCALL_TO_FLUSH_TLB,        NA,   0, FALSE) \
FLAG( 404,  0, EAX,  2,  1, USE_HYPERCALL_FOR_TLB_SHOOTDOWN,   NA,   0, FALSE) \
FLAG( 404,  0, EAX,  3,  1, USE_MSRS_FOR_EOI_ICR_TPR,          NA,   0, FALSE) \
FLAG( 404,  0, EAX,  4,  1, USE_MSR_FOR_RESET,                 NA,   0, FALSE) \
FLAG( 404,  0, EAX,  5,  1, USE_RELAXED_TIMING,                NA,   0, FALSE) \
FLAG( 404,  0, EAX,  6,  1, USE_DMA_REMAPPING,                 NA,   0, FALSE) \
FLAG( 404,  0, EAX,  7,  1, USE_INTERRUPT_REMAPPING,           NA,   0, FALSE) \
FLAG( 404,  0, EAX,  8,  1, USE_X2APIC,                        NA,   0, FALSE) \
FLAG( 404,  0, EAX,  9,  1, DEPRECATE_AUTOEOI,                 NA,   0, FALSE) \
FIELD(404,  0, EBX,  0, 32, SPINLOCK_RETRIES,                  NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_405                                             \
FIELD(405,  0, EAX,  0, 32, MAX_VCPU,                          NA,   0, FALSE) \
FIELD(405,  0, EBX,  0, 32, MAX_LCPU,                          NA,   0, FALSE) \
FIELD(405,  0, ECX,  0, 32, MAX_REMAPPABLE_VECTORS,            NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_406                                             \
FLAG( 406,  0, EAX,  0,  1, APIC_OVERLAY_ASSIST,               NA,   0, FALSE) \
FLAG( 406,  0, EAX,  1,  1, MSR_BITMAPS,                       NA,   0, FALSE) \
FLAG( 406,  0, EAX,  2,  1, ARCH_PMCS,                         NA,   0, FALSE) \
FLAG( 406,  0, EAX,  3,  1, SLAT,                              NA,   0, FALSE) \
FLAG( 406,  0, EAX,  4,  1, DMA_REMAPPING,                     NA,   0, FALSE) \
FLAG( 406,  0, EAX,  5,  1, INTERRUPT_REMAPPING,               NA,   0, FALSE) \
FLAG( 406,  0, EAX,  6,  1, MEMORY_PATROL_SCRUBBER,            NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_410                                             \
FIELD(410,  0, EAX,  0, 32, TSC_HZ,                            NA,   0, FALSE) \
FIELD(410,  0, EBX,  0, 32, APICBUS_HZ,                        NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_80                                              \
FIELD( 80,  0, EAX,  0, 32, NUM_EXT_LEVELS,                    NA,   0, FALSE) \
FIELD( 80,  0, EBX,  0, 32, LEAF80_VENDOR1,                    NA,   0, FALSE) \
FIELD( 80,  0, ECX,  0, 32, LEAF80_VENDOR3,                    NA,   0, FALSE) \
FIELD( 80,  0, EDX,  0, 32, LEAF80_VENDOR2,                    NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_81                                              \
FIELD( 81,  0, EAX,  0, 32, UNKNOWN81EAX,                      ANY,  4, FALSE) \
FIELD( 81,  0, EAX,  0,  4, LEAF81_STEPPING,                   ANY,  4, FALSE) \
FIELD( 81,  0, EAX,  4,  4, LEAF81_MODEL,                      ANY,  4, FALSE) \
FIELD( 81,  0, EAX,  8,  4, LEAF81_FAMILY,                     ANY,  4, FALSE) \
FIELD( 81,  0, EAX, 12,  2, LEAF81_TYPE,                       ANY,  4, FALSE) \
FIELD( 81,  0, EAX, 16,  4, LEAF81_EXTENDED_MODEL,             ANY,  4, FALSE) \
FIELD( 81,  0, EAX, 20,  8, LEAF81_EXTENDED_FAMILY,            ANY,  4, FALSE) \
FIELD( 81,  0, EBX,  0, 32, UNKNOWN81EBX,                      ANY,  4, FALSE) \
FIELD( 81,  0, EBX,  0, 16, LEAF81_BRAND_ID,                   ANY,  4, FALSE) \
FIELD( 81,  0, EBX, 16, 16, UNDEF,                             ANY,  4, FALSE) \
FLAG(  81,  0, ECX,  0,  1, LAHF64,                            YES,  4, TRUE)  \
FLAG(  81,  0, ECX,  1,  1, CMPLEGACY,                         ANY,  9, FALSE) \
FLAG(  81,  0, ECX,  2,  1, SVM,                               YES,  7, FALSE) \
FLAG(  81,  0, ECX,  3,  1, EXTAPICSPC,                        YES,  4, FALSE) \
FLAG(  81,  0, ECX,  4,  1, CR8AVAIL,                          YES,  4, FALSE) \
FLAG(  81,  0, ECX,  5,  1, ABM,                               YES,  7, TRUE)  \
FLAG(  81,  0, ECX,  6,  1, SSE4A,                             YES,  4, TRUE)  \
FLAG(  81,  0, ECX,  7,  1, MISALIGNED_SSE,                    YES,  4, TRUE)  \
FLAG(  81,  0, ECX,  8,  1, 3DNPREFETCH,                       YES,  4, TRUE)  \
FLAG(  81,  0, ECX,  9,  1, OSVW,                              ANY,  8, FALSE) \
FLAG(  81,  0, ECX, 10,  1, IBS,                               NO,   0, FALSE) \
FLAG(  81,  0, ECX, 11,  1, XOP,                               YES,  8, TRUE)  \
FLAG(  81,  0, ECX, 12,  1, SKINIT,                            NO,   0, FALSE) \
FLAG(  81,  0, ECX, 13,  1, WATCHDOG,                          NO,   0, FALSE) \
FLAG(  81,  0, ECX, 15,  1, LWP,                               NO,   0, FALSE) \
FLAG(  81,  0, ECX, 16,  1, FMA4,                              YES,  8, TRUE)  \
FLAG(  81,  0, ECX, 17,  1, TCE,                               NO,   0, FALSE) \
FLAG(  81,  0, ECX, 19,  1, NODEID_MSR,                        NO,   0, FALSE) \
FLAG(  81,  0, ECX, 21,  1, TBM,                               YES,  9, TRUE)  \
FLAG(  81,  0, ECX, 22,  1, TOPOLOGY,                          NO,   0, FALSE) \
FLAG(  81,  0, ECX, 23,  1, PERFCORE,                          ANY,  4, TRUE)  \
FLAG(  81,  0, ECX, 24,  1, PERFNB,                            NO,   0, FALSE) \
FLAG(  81,  0, ECX, 26,  1, DATABK,                            NO,   0, FALSE) \
FLAG(  81,  0, ECX, 27,  1, PERFTSC,                           NO,   0, FALSE) \
FLAG(  81,  0, ECX, 28,  1, PERFL3,                            NO,   0, FALSE) \
FLAG(  81,  0, ECX, 29,  1, MWAITX,                            NO,   0, FALSE) \
FLAG(  81,  0, EDX,  0,  1, LEAF81_FPU,                        YES,  4, TRUE)  \
FLAG(  81,  0, EDX,  1,  1, LEAF81_VME,                        YES,  4, FALSE) \
FLAG(  81,  0, EDX,  2,  1, LEAF81_DE,                         YES,  4, FALSE) \
FLAG(  81,  0, EDX,  3,  1, LEAF81_PSE,                        YES,  4, FALSE) \
FLAG(  81,  0, EDX,  4,  1, LEAF81_TSC,                        YES,  4, TRUE)  \
FLAG(  81,  0, EDX,  5,  1, LEAF81_MSR,                        YES,  4, FALSE) \
FLAG(  81,  0, EDX,  6,  1, LEAF81_PAE,                        YES,  4, FALSE) \
FLAG(  81,  0, EDX,  7,  1, LEAF81_MCE,                        YES,  4, FALSE) \
FLAG(  81,  0, EDX,  8,  1, LEAF81_CX8,                        YES,  4, TRUE)  \
FLAG(  81,  0, EDX,  9,  1, LEAF81_APIC,                       ANY,  4, FALSE) \
FLAG(  81,  0, EDX, 11,  1, SYSC,                              ANY,  4, TRUE)  \
FLAG(  81,  0, EDX, 12,  1, LEAF81_MTRR,                       YES,  4, FALSE) \
FLAG(  81,  0, EDX, 13,  1, LEAF81_PGE,                        YES,  4, FALSE) \
FLAG(  81,  0, EDX, 14,  1, LEAF81_MCA,                        YES,  4, FALSE) \
FLAG(  81,  0, EDX, 15,  1, LEAF81_CMOV,                       YES,  4, TRUE)  \
FLAG(  81,  0, EDX, 16,  1, LEAF81_PAT,                        YES,  4, FALSE) \
FLAG(  81,  0, EDX, 17,  1, LEAF81_PSE36,                      YES,  4, FALSE) \
FLAG(  81,  0, EDX, 20,  1, NX,                                YES,  4, FALSE) \
FLAG(  81,  0, EDX, 22,  1, MMXEXT,                            YES,  4, TRUE)  \
FLAG(  81,  0, EDX, 23,  1, LEAF81_MMX,                        YES,  4, TRUE)  \
FLAG(  81,  0, EDX, 24,  1, LEAF81_FXSR,                       YES,  4, TRUE)  \
FLAG(  81,  0, EDX, 25,  1, FFXSR,                             YES,  4, FALSE) \
FLAG(  81,  0, EDX, 26,  1, PDPE1GB,                           YES,  7, FALSE) \
FLAG(  81,  0, EDX, 27,  1, RDTSCP,                            YES,  4, TRUE)  \
FLAG(  81,  0, EDX, 29,  1, LM,                                YES,  4, FALSE) \
FLAG(  81,  0, EDX, 30,  1, 3DNOWPLUS,                         YES,  4, TRUE)  \
FLAG(  81,  0, EDX, 31,  1, 3DNOW,                             YES,  4, TRUE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_82                                              \
FIELD( 82,  0, EAX,  0, 32, LEAF82_BRAND_STRING_EAX,           NA,   0, FALSE) \
FIELD( 82,  0, EBX,  0, 32, LEAF82_BRAND_STRING_EBX,           NA,   0, FALSE) \
FIELD( 82,  0, ECX,  0, 32, LEAF82_BRAND_STRING_ECX,           NA,   0, FALSE) \
FIELD( 82,  0, EDX,  0, 32, LEAF82_BRAND_STRING_EDX,           NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_83                                              \
FIELD( 83,  0, EAX,  0, 32, LEAF83_BRAND_STRING_EAX,           NA,   0, FALSE) \
FIELD( 83,  0, EBX,  0, 32, LEAF83_BRAND_STRING_EBX,           NA,   0, FALSE) \
FIELD( 83,  0, ECX,  0, 32, LEAF83_BRAND_STRING_ECX,           NA,   0, FALSE) \
FIELD( 83,  0, EDX,  0, 32, LEAF83_BRAND_STRING_EDX,           NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_84                                              \
FIELD( 84,  0, EAX,  0, 32, LEAF84_BRAND_STRING_EAX,           NA,   0, FALSE) \
FIELD( 84,  0, EBX,  0, 32, LEAF84_BRAND_STRING_EBX,           NA,   0, FALSE) \
FIELD( 84,  0, ECX,  0, 32, LEAF84_BRAND_STRING_ECX,           NA,   0, FALSE) \
FIELD( 84,  0, EDX,  0, 32, LEAF84_BRAND_STRING_EDX,           NA,   0, FALSE)

/*    LEVEL, REG, POS, SIZE, NAME,                        MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_85                                              \
FIELD( 85,  0, EAX,  0,  8, ITLB_ENTRIES_2M4M_PGS,             NA,   0, FALSE) \
FIELD( 85,  0, EAX,  8,  8, ITLB_ASSOC_2M4M_PGS,               NA,   0, FALSE) \
FIELD( 85,  0, EAX, 16,  8, DTLB_ENTRIES_2M4M_PGS,             NA,   0, FALSE) \
FIELD( 85,  0, EAX, 24,  8, DTLB_ASSOC_2M4M_PGS,               NA,   0, FALSE) \
FIELD( 85,  0, EBX,  0,  8, ITLB_ENTRIES_4K_PGS,               NA,   0, FALSE) \
FIELD( 85,  0, EBX,  8,  8, ITLB_ASSOC_4K_PGS,                 NA,   0, FALSE) \
FIELD( 85,  0, EBX, 16,  8, DTLB_ENTRIES_4K_PGS,               NA,   0, FALSE) \
FIELD( 85,  0, EBX, 24,  8, DTLB_ASSOC_4K_PGS,                 NA,   0, FALSE) \
FIELD( 85,  0, ECX,  0,  8, L1_DCACHE_LINE_SIZE,               NA,   0, FALSE) \
FIELD( 85,  0, ECX,  8,  8, L1_DCACHE_LINES_PER_TAG,           NA,   0, FALSE) \
FIELD( 85,  0, ECX, 16,  8, L1_DCACHE_ASSOC,                   NA,   0, FALSE) \
FIELD( 85,  0, ECX, 24,  8, L1_DCACHE_SIZE,                    NA,   0, FALSE) \
FIELD( 85,  0, EDX,  0,  8, L1_ICACHE_LINE_SIZE,               NA,   0, FALSE) \
FIELD( 85,  0, EDX,  8,  8, L1_ICACHE_LINES_PER_TAG,           NA,   0, FALSE) \
FIELD( 85,  0, EDX, 16,  8, L1_ICACHE_ASSOC,                   NA,   0, FALSE) \
FIELD( 85,  0, EDX, 24,  8, L1_ICACHE_SIZE,                    NA,   0, FALSE)

/*    LEVEL, REG, POS, SIZE, NAME,                        MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_86                                              \
FIELD( 86,  0, EAX,  0, 12, L2_ITLB_ENTRIES_2M4M_PGS,          NA,   0, FALSE) \
FIELD( 86,  0, EAX, 12,  4, L2_ITLB_ASSOC_2M4M_PGS,            NA,   0, FALSE) \
FIELD( 86,  0, EAX, 16, 12, L2_DTLB_ENTRIES_2M4M_PGS,          NA,   0, FALSE) \
FIELD( 86,  0, EAX, 28,  4, L2_DTLB_ASSOC_2M4M_PGS,            NA,   0, FALSE) \
FIELD( 86,  0, EBX,  0, 12, L2_ITLB_ENTRIES_4K_PGS,            NA,   0, FALSE) \
FIELD( 86,  0, EBX, 12,  4, L2_ITLB_ASSOC_4K_PGS,              NA,   0, FALSE) \
FIELD( 86,  0, EBX, 16, 12, L2_DTLB_ENTRIES_4K_PGS,            NA,   0, FALSE) \
FIELD( 86,  0, EBX, 28,  4, L2_DTLB_ASSOC_4K_PGS,              NA,   0, FALSE) \
FIELD( 86,  0, ECX,  0,  8, L2CACHE_LINE,                      NA,   0, FALSE) \
FIELD( 86,  0, ECX,  8,  4, L2CACHE_LINE_PER_TAG,              NA,   0, FALSE) \
FIELD( 86,  0, ECX, 12,  4, L2CACHE_WAYS,                      NA,   0, FALSE) \
FIELD( 86,  0, ECX, 16, 16, L2CACHE_SIZE,                      NA,   0, FALSE) \
FIELD( 86,  0, EDX,  0,  8, L3CACHE_LINE,                      NA,   0, FALSE) \
FIELD( 86,  0, EDX,  8,  4, L3CACHE_LINE_PER_TAG,              NA,   0, FALSE) \
FIELD( 86,  0, EDX, 12,  4, L3CACHE_WAYS,                      NA,   0, FALSE) \
FIELD( 86,  0, EDX, 18, 14, L3CACHE_SIZE,                      NA,   0, FALSE)

/*    LEVEL, REG, POS, SIZE, NAME,                        MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_87                                              \
FLAG(  87,  0, EBX,  0,  1, MCA_OVERFLOW_RECOV,                NA,   0, FALSE) \
FLAG(  87,  0, EBX,  1,  1, SUCCOR,                            NA,   0, FALSE) \
FLAG(  87,  0, EBX,  2,  1, HWA,                               NA,   0, FALSE) \
FLAG(  87,  0, EBX,  3,  1, SCALABLE_MCA,                      NA,   0, FALSE) \
FLAG(  87,  0, EBX,  4,  1, PFEH_SUPPORT_PRESENT,              NA,   0, FALSE) \
FLAG(  87,  0, EDX,  0,  1, TS,                                NA,   0, FALSE) \
FLAG(  87,  0, EDX,  1,  1, FID,                               NA,   0, FALSE) \
FLAG(  87,  0, EDX,  2,  1, VID,                               NA,   0, FALSE) \
FLAG(  87,  0, EDX,  3,  1, TTP,                               NA,   0, FALSE) \
FLAG(  87,  0, EDX,  4,  1, LEAF87_TM,                         NA,   0, FALSE) \
FLAG(  87,  0, EDX,  5,  1, STC,                               NA,   0, FALSE) \
FLAG(  87,  0, EDX,  6,  1, 100MHZSTEPS,                       NA,   0, FALSE) \
FLAG(  87,  0, EDX,  7,  1, HWPSTATE,                          NA,   0, FALSE) \
FLAG(  87,  0, EDX,  8,  1, TSC_INVARIANT,                     NA,   0, FALSE) \
FLAG(  87,  0, EDX,  9,  1, CORE_PERF_BOOST,                   NA,   0, FALSE)

#define CPUID_88_EBX_14 \
FLAG(  88,  0, EBX, 14,  1, LEAF88_RSVD1,                       NO,  0, FALSE)
#define CPUID_88_EBX_15 \
FLAG(  88,  0, EBX, 15,  1, LEAF88_RSVD2,                       NO,  0, FALSE)
#define CPUID_88_EBX_16 \
FLAG(  88,  0, EBX, 16,  1, LEAF88_RSVD3,                       NO,  0, FALSE)
#define CPUID_88_EBX_17 \
FLAG(  88,  0, EBX, 17,  1, LEAF88_RSVD4,                       NO,  0, FALSE)
#define CPUID_88_EBX_18 \
FLAG(  88,  0, EBX, 18,  1, LEAF88_RSVD5,                       NO,  0, FALSE)

/*    LEVEL, REG, POS, SIZE, NAME,                        MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_88                                              \
FIELD( 88,  0, EAX,  0,  8, PHYS_BITS,                         YES,  4, FALSE) \
FIELD( 88,  0, EAX,  8,  8, VIRT_BITS,                         YES,  4, FALSE) \
FIELD( 88,  0, EAX, 16,  8, GUEST_PHYS_ADDR_SZ,                YES,  8, FALSE) \
FLAG(  88,  0, EBX,  0,  1, CLZERO,                            YES, 14, TRUE)  \
FLAG(  88,  0, EBX,  1,  1, IRPERF,                            NO,   0, FALSE) \
FLAG(  88,  0, EBX,  2,  1, XSAVE_ERR_PTR,                     NO,   0, FALSE) \
FLAG(  88,  0, EBX, 12,  1, LEAF88_IBPB,                       ANY,  9, FALSE) \
CPUID_88_EBX_14 \
CPUID_88_EBX_15 \
CPUID_88_EBX_16 \
CPUID_88_EBX_17 \
CPUID_88_EBX_18 \
FIELD( 88,  0, ECX,  0,  8, LEAF88_CORE_COUNT,                 YES,  4, FALSE) \
FIELD( 88,  0, ECX, 12,  4, APICID_COREID_SIZE,                YES,  7, FALSE) \
FIELD( 88,  0, ECX, 16,  2, PERFTSC_SIZE,                      NO,   0, FALSE)

#define CPUID_8A_EDX_11 \
FLAG(  8A,  0, EDX, 11,  1, SVMEDX_RSVD1,                      NO,   0, FALSE)
#define CPUID_8A_EDX_14 \
FLAG(  8A,  0, EDX, 14,  1, SVMEDX_RSVD2,                      NO,   0, FALSE)

/*    LEVEL, REG, POS, SIZE, NAME,                        MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_8A                                              \
FIELD( 8A,  0, EAX,  0,  8, SVM_REVISION,                      YES,  4, FALSE) \
FLAG(  8A,  0, EAX,  8,  1, SVM_HYPERVISOR,                    NO,   0, FALSE) \
FIELD( 8A,  0, EAX,  9, 23, SVMEAX_RSVD,                       NO,   0, FALSE) \
FIELD( 8A,  0, EBX,  0, 32, SVM_NUM_ASIDS,                     YES,  7, FALSE) \
FIELD( 8A,  0, ECX,  0, 32, SVMECX_RSVD,                       NO,   0, FALSE) \
FLAG(  8A,  0, EDX,  0,  1, SVM_NPT,                           YES,  7, FALSE) \
FLAG(  8A,  0, EDX,  1,  1, SVM_LBR,                           NO,   0, FALSE) \
FLAG(  8A,  0, EDX,  2,  1, SVM_LOCK,                          ANY,  7, FALSE) \
FLAG(  8A,  0, EDX,  3,  1, SVM_NRIP,                          YES,  7, FALSE) \
FLAG(  8A,  0, EDX,  4,  1, SVM_TSC_RATE_MSR,                  NO,   0, FALSE) \
FLAG(  8A,  0, EDX,  5,  1, SVM_VMCB_CLEAN,                    YES,  7, FALSE) \
FLAG(  8A,  0, EDX,  6,  1, SVM_FLUSH_BY_ASID,                 YES,  7, FALSE) \
FLAG(  8A,  0, EDX,  7,  1, SVM_DECODE_ASSISTS,                YES,  7, FALSE) \
FIELD( 8A,  0, EDX,  8,  2, SVMEDX_RSVD0,                      NO,   0, FALSE) \
FLAG(  8A,  0, EDX, 10,  1, SVM_PAUSE_FILTER,                  NO,   0, FALSE) \
CPUID_8A_EDX_11 \
FLAG(  8A,  0, EDX, 12,  1, SVM_PAUSE_THRESHOLD,               NO,   0, FALSE) \
FLAG(  8A,  0, EDX, 13,  1, SVM_AVIC,                          NO,   0, FALSE) \
CPUID_8A_EDX_14 \
FLAG(  8A,  0, EDX, 15,  1, SVM_V_VMSAVE_VMLOAD,               NO,   0, FALSE) \
FLAG(  8A,  0, EDX, 16,  1, SVM_VGIF,                          NO,   0, FALSE) \
FIELD( 8A,  0, EDX, 17, 15, SVMEDX_RSVD,                       NO,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_819                                             \
FIELD(819,  0, EAX,  0, 12, L1_ITLB_ENTRIES_1G_PGS,            NA,   0, FALSE) \
FIELD(819,  0, EAX, 12,  4, L1_ITLB_ASSOC_1G_PGS,              NA,   0, FALSE) \
FIELD(819,  0, EAX, 16, 12, L1_DTLB_ENTRIES_1G_PGS,            NA,   0, FALSE) \
FIELD(819,  0, EAX, 28,  4, L1_DTLB_ASSOC_1G_PGS,              NA,   0, FALSE) \
FIELD(819,  0, EBX,  0, 12, L2_ITLB_ENTRIES_1G_PGS,            NA,   0, FALSE) \
FIELD(819,  0, EBX, 12,  4, L2_ITLB_ASSOC_1G_PGS,              NA,   0, FALSE) \
FIELD(819,  0, EBX, 16, 12, L2_DTLB_ENTRIES_1G_PGS,            NA,   0, FALSE) \
FIELD(819,  0, EBX, 28,  4, L2_DTLB_ASSOC_1G_PGS,              NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_81A                                             \
FLAG( 81A,  0, EAX,  0,  1, FP128,                             NA,   0, FALSE) \
FLAG( 81A,  0, EAX,  1,  1, MOVU,                              NA,   0, FALSE) \
FLAG( 81A,  0, EAX,  2,  1, FP256,                             NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_81B                                             \
FLAG( 81B,  0, EAX,  0,  1, IBS_FFV,                           NA,   0, FALSE) \
FLAG( 81B,  0, EAX,  1,  1, IBS_FETCHSAM,                      NA,   0, FALSE) \
FLAG( 81B,  0, EAX,  2,  1, IBS_OPSAM,                         NA,   0, FALSE) \
FLAG( 81B,  0, EAX,  3,  1, RW_OPCOUNT,                        NA,   0, FALSE) \
FLAG( 81B,  0, EAX,  4,  1, OPCOUNT,                           NA,   0, FALSE) \
FLAG( 81B,  0, EAX,  5,  1, BRANCH_TARGET_ADDR,                NA,   0, FALSE) \
FLAG( 81B,  0, EAX,  6,  1, OPCOUNT_EXT,                       NA,   0, FALSE) \
FLAG( 81B,  0, EAX,  7,  1, RIP_INVALID_CHECK,                 NA,   0, FALSE) \
FLAG( 81B,  0, EAX,  8,  1, OP_BRN_FUSE,                       NA,   0, FALSE) \
FLAG( 81B,  0, EAX,  9,  1, IBS_FETCH_CTL_EXTD,                NA,   0, FALSE) \
FLAG( 81B,  0, EAX, 10,  1, IBS_OP_DATA4,                      NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_81C                                             \
FLAG( 81C,  0, EAX,  0,  1, LWP_AVAIL,                         NA,   0, FALSE) \
FLAG( 81C,  0, EAX,  1,  1, LWP_VAL_AVAIL,                     NA,   0, FALSE) \
FLAG( 81C,  0, EAX,  2,  1, LWP_IRE_AVAIL,                     NA,   0, FALSE) \
FLAG( 81C,  0, EAX,  3,  1, LWP_BRE_AVAIL,                     NA,   0, FALSE) \
FLAG( 81C,  0, EAX,  4,  1, LWP_DME_AVAIL,                     NA,   0, FALSE) \
FLAG( 81C,  0, EAX,  5,  1, LWP_CNH_AVAIL,                     NA,   0, FALSE) \
FLAG( 81C,  0, EAX,  6,  1, LWP_RNH_AVAIL,                     NA,   0, FALSE) \
FLAG( 81C,  0, EAX, 29,  1, LWP_CONT_AVAIL,                    NA,   0, FALSE) \
FLAG( 81C,  0, EAX, 30,  1, LWP_PTSC_AVAIL,                    NA,   0, FALSE) \
FLAG( 81C,  0, EAX, 31,  1, LWP_INT_AVAIL,                     NA,   0, FALSE) \
FIELD(81C,  0, EBX,  0,  8, LWP_CB_SIZE,                       NA,   0, FALSE) \
FIELD(81C,  0, EBX,  8,  8, LWP_EVENT_SIZE,                    NA,   0, FALSE) \
FIELD(81C,  0, EBX, 16,  8, LWP_MAX_EVENTS,                    NA,   0, FALSE) \
FIELD(81C,  0, EBX, 24,  8, LWP_EVENT_OFFSET,                  NA,   0, FALSE) \
FIELD(81C,  0, ECX,  0,  4, LWP_LATENCY_MAX,                   NA,   0, FALSE) \
FLAG( 81C,  0, ECX,  5,  1, LWP_DATA_ADDR_VALID,               NA,   0, FALSE) \
FIELD(81C,  0, ECX,  6,  3, LWP_LATENCY_ROUND,                 NA,   0, FALSE) \
FIELD(81C,  0, ECX,  9,  7, LWP_VERSION,                       NA,   0, FALSE) \
FIELD(81C,  0, ECX, 16,  8, LWP_MIN_BUF_SIZE,                  NA,   0, FALSE) \
FLAG( 81C,  0, ECX, 28,  1, LWP_BRANCH_PRED,                   NA,   0, FALSE) \
FLAG( 81C,  0, ECX, 29,  1, LWP_IP_FILTERING,                  NA,   0, FALSE) \
FLAG( 81C,  0, ECX, 30,  1, LWP_CACHE_LEVEL,                   NA,   0, FALSE) \
FLAG( 81C,  0, ECX, 31,  1, LWP_CACHE_LATENCY,                 NA,   0, FALSE) \
FLAG( 81C,  0, EDX,  0,  1, LWP_SUPPORTED,                     NA,   0, FALSE) \
FLAG( 81C,  0, EDX,  1,  1, LWP_VAL_SUPPORTED,                 NA,   0, FALSE) \
FLAG( 81C,  0, EDX,  2,  1, LWP_IRE_SUPPORTED,                 NA,   0, FALSE) \
FLAG( 81C,  0, EDX,  3,  1, LWP_BRE_SUPPORTED,                 NA,   0, FALSE) \
FLAG( 81C,  0, EDX,  4,  1, LWP_DME_SUPPORTED,                 NA,   0, FALSE) \
FLAG( 81C,  0, EDX,  5,  1, LWP_CNH_SUPPORTED,                 NA,   0, FALSE) \
FLAG( 81C,  0, EDX,  6,  1, LWP_RNH_SUPPORTED,                 NA,   0, FALSE) \
FLAG( 81C,  0, EDX, 29,  1, LWP_CONT_SUPPORTED,                NA,   0, FALSE) \
FLAG( 81C,  0, EDX, 30,  1, LWP_PTSC_SUPPORTED,                NA,   0, FALSE) \
FLAG( 81C,  0, EDX, 31,  1, LWP_INT_SUPPORTED,                 NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_81D                                             \
FIELD(81D,  0, EAX,  0,  5, LEAF81D_CACHE_TYPE,                NA,   0, FALSE) \
FIELD(81D,  0, EAX,  5,  3, LEAF81D_CACHE_LEVEL,               NA,   0, FALSE) \
FLAG( 81D,  0, EAX,  8,  1, LEAF81D_CACHE_SELF_INIT,           NA,   0, FALSE) \
FLAG( 81D,  0, EAX,  9,  1, LEAF81D_CACHE_FULLY_ASSOC,         NA,   0, FALSE) \
FIELD(81D,  0, EAX, 14, 12, LEAF81D_NUM_SHARING_CACHE,         NA,   0, FALSE) \
FIELD(81D,  0, EBX,  0, 12, LEAF81D_CACHE_LINE_SIZE,           NA,   0, FALSE) \
FIELD(81D,  0, EBX, 12, 10, LEAF81D_CACHE_PHYS_PARTITIONS,     NA,   0, FALSE) \
FIELD(81D,  0, EBX, 22, 10, LEAF81D_CACHE_WAYS,                NA,   0, FALSE) \
FIELD(81D,  0, ECX,  0, 32, LEAF81D_CACHE_NUM_SETS,            NA,   0, FALSE) \
FLAG( 81D,  0, EDX,  0,  1, LEAF81D_CACHE_WBINVD,              NA,   0, FALSE) \
FLAG( 81D,  0, EDX,  1,  1, LEAF81D_CACHE_INCLUSIVE,           NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_81E                                             \
FIELD(81E,  0, EAX,  0, 32, EXTENDED_APICID,                   NA,   0, FALSE) \
FIELD(81E,  0, EBX,  0,  8, COMPUTE_UNIT_ID,                   NA,   0, FALSE) \
FIELD(81E,  0, EBX,  8,  2, CORES_PER_COMPUTE_UNIT,            NA,   0, FALSE) \
FIELD(81E,  0, ECX,  0,  8, NODEID_VAL,                        NA,   0, FALSE) \
FIELD(81E,  0, ECX,  8,  3, NODES_PER_PKG,                     NA,   0, FALSE)

/*    LEVEL, SUB-LEVEL, REG, POS, SIZE, NAME,             MON SUPP, HWV, CPL3 */
#define CPUID_FIELD_DATA_LEVEL_81F                                             \
FLAG( 81F,  0, EAX,  0,  1, SME,                               NO,   0, FALSE) \
FLAG( 81F,  0, EAX,  1,  1, SEV,                               NO,   0, FALSE) \
FLAG( 81F,  0, EAX,  2,  1, PAGE_FLUSH_MSR,                    NO,   0, FALSE) \
FLAG( 81F,  0, EAX,  3,  1, SEV_ES,                            NO,   0, FALSE) \
FIELD(81F,  0, EBX,  0,  5, SME_PAGE_TABLE_BIT_NUM,            NO,   0, FALSE) \
FIELD(81F,  0, EBX,  6,  6, SME_PHYS_ADDR_SPACE_REDUCTION,     NO,   0, FALSE) \
FIELD(81F,  0, ECX,  0, 32, NUM_ENCRYPTED_GUESTS,              NO,   0, FALSE) \
FIELD(81F,  0, EDX,  0, 32, SEV_MIN_ASID,                      NO,   0, FALSE)

#define INTEL_CPUID_FIELD_DATA

#define AMD_CPUID_FIELD_DATA

#define CPUID_FIELD_DATA                                              \
   CPUID_FIELD_DATA_LEVEL_0                                           \
   CPUID_FIELD_DATA_LEVEL_1                                           \
   CPUID_FIELD_DATA_LEVEL_2                                           \
   CPUID_FIELD_DATA_LEVEL_4                                           \
   CPUID_FIELD_DATA_LEVEL_5                                           \
   CPUID_FIELD_DATA_LEVEL_6                                           \
   CPUID_FIELD_DATA_LEVEL_7                                           \
   CPUID_FIELD_DATA_LEVEL_A                                           \
   CPUID_FIELD_DATA_LEVEL_B                                           \
   CPUID_FIELD_DATA_LEVEL_D                                           \
   CPUID_FIELD_DATA_LEVEL_F                                           \
   CPUID_FIELD_DATA_LEVEL_10                                          \
   CPUID_FIELD_DATA_LEVEL_12                                          \
   CPUID_FIELD_DATA_LEVEL_14                                          \
   CPUID_FIELD_DATA_LEVEL_15                                          \
   CPUID_FIELD_DATA_LEVEL_16                                          \
   CPUID_FIELD_DATA_LEVEL_17                                          \
   CPUID_FIELD_DATA_LEVEL_400                                         \
   CPUID_FIELD_DATA_LEVEL_401                                         \
   CPUID_FIELD_DATA_LEVEL_402                                         \
   CPUID_FIELD_DATA_LEVEL_403                                         \
   CPUID_FIELD_DATA_LEVEL_404                                         \
   CPUID_FIELD_DATA_LEVEL_405                                         \
   CPUID_FIELD_DATA_LEVEL_406                                         \
   CPUID_FIELD_DATA_LEVEL_410                                         \
   CPUID_FIELD_DATA_LEVEL_80                                          \
   CPUID_FIELD_DATA_LEVEL_81                                          \
   CPUID_FIELD_DATA_LEVEL_82                                          \
   CPUID_FIELD_DATA_LEVEL_83                                          \
   CPUID_FIELD_DATA_LEVEL_84                                          \
   CPUID_FIELD_DATA_LEVEL_85                                          \
   CPUID_FIELD_DATA_LEVEL_86                                          \
   CPUID_FIELD_DATA_LEVEL_87                                          \
   CPUID_FIELD_DATA_LEVEL_88                                          \
   CPUID_FIELD_DATA_LEVEL_8A                                          \
   CPUID_FIELD_DATA_LEVEL_819                                         \
   CPUID_FIELD_DATA_LEVEL_81A                                         \
   CPUID_FIELD_DATA_LEVEL_81B                                         \
   CPUID_FIELD_DATA_LEVEL_81C                                         \
   CPUID_FIELD_DATA_LEVEL_81D                                         \
   CPUID_FIELD_DATA_LEVEL_81E                                         \
   CPUID_FIELD_DATA_LEVEL_81F                                         \
   INTEL_CPUID_FIELD_DATA                                             \
   AMD_CPUID_FIELD_DATA

/*
 * Define all field and flag values as an enum.  The result is a full
 * set of values taken from the table above in the form:
 *
 * CPUID_<name>_MASK  == mask for feature/field
 * CPUID_<name>_SHIFT == offset of field
 *
 * e.g. - CPUID_VIRT_BITS_MASK  = 0xff00
 *      - CPUID_VIRT_BITS_SHIFT = 8
 */
#define VMW_BIT_MASK(shift)  (0xffffffffu >> (32 - shift))

#define FIELD(lvl, ecxIn, reg, bitpos, size, name, s, hwv, c3) \
   CPUID_##name##_SHIFT        = bitpos,                       \
   CPUID_##name##_MASK         = VMW_BIT_MASK(size) << bitpos, \
   CPUID_INTERNAL_SHIFT_##name = bitpos,                       \
   CPUID_INTERNAL_MASK_##name  = VMW_BIT_MASK(size) << bitpos, \
   CPUID_INTERNAL_REG_##name   = CPUID_REG_##reg,              \
   CPUID_INTERNAL_EAXIN_##name = CPUID_LEVEL_VAL_##lvl,        \
   CPUID_INTERNAL_ECXIN_##name = ecxIn,                        \
   CPUID_INTERNAL_HWV_##name   = hwv,

#define FLAG FIELD

enum {
   /* Define data for every CPUID field we have */
   CPUID_FIELD_DATA
};
#undef VMW_BIT_MASK
#undef FIELD
#undef FLAG

/*
 * CPUID_MASK --
 * CPUID_SHIFT --
 * CPUID_ISSET --
 * CPUID_GET --
 * CPUID_SET --
 * CPUID_CLEAR --
 * CPUID_SETTO --
 *
 * Accessor macros for all CPUID consts/fields/flags.  Level and reg are not
 * required, but are used to force compile-time asserts which help verify that
 * the flag is being used on the right CPUID input and result register.
 *
 * Note: ASSERT_ON_COMPILE is duplicated rather than factored into its own
 * macro, because token concatenation does not work as expected if an input is
 * #defined (e.g. APIC) when macros are nested.  Also, compound statements
 * within parenthes is a GCC extension, so we must use runtime asserts with
 * other compilers.
 */

#if defined(__GNUC__) && !defined(__clang__)

#define CPUID_MASK(eaxIn, reg, flag)                                    \
   ({                                                                   \
      ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##flag &&         \
              CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag);  \
      CPUID_INTERNAL_MASK_##flag;                                       \
   })

#define CPUID_SHIFT(eaxIn, reg, flag)                                   \
   ({                                                                   \
      ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##flag &&         \
              CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag);  \
      CPUID_INTERNAL_SHIFT_##flag;                                      \
   })

#define CPUID_ISSET(eaxIn, reg, flag, data)                             \
   ({                                                                   \
      ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##flag &&         \
              CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag);  \
      (((data) & CPUID_INTERNAL_MASK_##flag) != 0);                     \
   })

#define CPUID_GET(eaxIn, reg, field, data)                              \
   ({                                                                   \
      ASSERT_ON_COMPILE(eaxIn == CPUID_INTERNAL_EAXIN_##field &&        \
              CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##field); \
      (((uint32)(data) & CPUID_INTERNAL_MASK_##field) >>                \
       CPUID_INTERNAL_SHIFT_##field);                                   \
   })

#else

/*
 * CPUIDCheck --
 *
 * Return val after verifying parameters.
 */

static INLINE uint32
CPUIDCheck(int32 eaxIn, int32 eaxInCheck,
           CpuidReg reg, CpuidReg regCheck, uint32 val)
{
   ASSERT(eaxIn == eaxInCheck && reg == regCheck);
   return val;
}

#define CPUID_MASK(eaxIn, reg, flag)                                    \
   CPUIDCheck(eaxIn, CPUID_INTERNAL_EAXIN_##flag,                       \
              CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##flag,     \
              CPUID_INTERNAL_MASK_##flag)

#define CPUID_SHIFT(eaxIn, reg, flag)                                   \
   CPUIDCheck(eaxIn, CPUID_INTERNAL_EAXIN_##flag,                       \
              CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##flag,     \
              CPUID_INTERNAL_SHIFT_##flag)

#define CPUID_ISSET(eaxIn, reg, flag, data)                             \
   (CPUIDCheck(eaxIn, CPUID_INTERNAL_EAXIN_##flag,                      \
               CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##flag,    \
               CPUID_INTERNAL_MASK_##flag & (data)) != 0)

#define CPUID_GET(eaxIn, reg, field, data)                              \
   CPUIDCheck(eaxIn, CPUID_INTERNAL_EAXIN_##field,                      \
              CPUID_REG_##reg, (CpuidReg)CPUID_INTERNAL_REG_##field,    \
              ((uint32)(data) & CPUID_INTERNAL_MASK_##field) >>         \
              CPUID_INTERNAL_SHIFT_##field)

#endif

#define CPUID_SET(eaxIn, reg, flag, dataPtr)                            \
   do {                                                                 \
      ASSERT_ON_COMPILE(                                                \
         (uint32)eaxIn   == (uint32)CPUID_INTERNAL_EAXIN_##flag &&      \
         CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag);       \
      *(dataPtr) |= CPUID_INTERNAL_MASK_##flag;                         \
   } while (0)

#define CPUID_CLEAR(eaxIn, reg, flag, dataPtr)                          \
   do {                                                                 \
      ASSERT_ON_COMPILE(                                                \
         (uint32)eaxIn   == (uint32)CPUID_INTERNAL_EAXIN_##flag &&      \
         CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##flag);       \
      *(dataPtr) &= ~CPUID_INTERNAL_MASK_##flag;                        \
   } while (0)

#define CPUID_SETTO(eaxIn, reg, field, dataPtr, val)                    \
   do {                                                                 \
      uint32 _v = val;                                                  \
      uint32 *_d = dataPtr;                                             \
      ASSERT_ON_COMPILE(                                                \
         (uint32)eaxIn   == (uint32)CPUID_INTERNAL_EAXIN_##field &&     \
         CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##field);      \
      *_d = (*_d & ~CPUID_INTERNAL_MASK_##field) |                      \
         (_v << CPUID_INTERNAL_SHIFT_##field);                          \
      ASSERT(_v == (*_d & CPUID_INTERNAL_MASK_##field) >>               \
             CPUID_INTERNAL_SHIFT_##field);                             \
   } while (0)

#define CPUID_SETTO_SAFE(eaxIn, reg, field, dataPtr, val)               \
   do {                                                                 \
      uint32 _v = val &                                                 \
         (CPUID_INTERNAL_MASK_##field >> CPUID_INTERNAL_SHIFT_##field); \
      uint32 *_d = dataPtr;                                             \
      ASSERT_ON_COMPILE(                                                \
         (uint32)eaxIn   == (uint32)CPUID_INTERNAL_EAXIN_##field &&     \
         CPUID_REG_##reg == (CpuidReg)CPUID_INTERNAL_REG_##field);      \
      *_d = (*_d & ~CPUID_INTERNAL_MASK_##field) |                      \
         (_v << CPUID_INTERNAL_SHIFT_##field);                          \
   } while (0)

/*
 * Definitions of various fields' values and more complicated
 * macros/functions for reading cpuid fields.
 */

#define CPUID_FAMILY_EXTENDED        15

/* Effective Intel CPU Families */
#define CPUID_FAMILY_486              4
#define CPUID_FAMILY_P5               5
#define CPUID_FAMILY_P6               6
#define CPUID_FAMILY_P4              15

/* Effective AMD CPU Families */
#define CPUID_FAMILY_5x86            0x4
#define CPUID_FAMILY_K5              0x5
#define CPUID_FAMILY_K6              0x5
#define CPUID_FAMILY_K7              0x6
#define CPUID_FAMILY_K8              0xf
#define CPUID_FAMILY_K8L             0x10
#define CPUID_FAMILY_K8MOBILE        0x11
#define CPUID_FAMILY_LLANO           0x12
#define CPUID_FAMILY_BOBCAT          0x14
#define CPUID_FAMILY_BULLDOZER       0x15  // BD PD SR EX
#define CPUID_FAMILY_KYOTO           0x16  // Note: Jaguar microarch
#define CPUID_FAMILY_ZEN             0x17

/* Effective VIA CPU Families */
#define CPUID_FAMILY_C7               6

/* Effective Hygon CPU Families. */
#define CPUID_FAMILY_DHYANA          0x18

/* Intel model information */
#define CPUID_MODEL_PPRO              1
#define CPUID_MODEL_PII_03            3
#define CPUID_MODEL_PII_05            5
#define CPUID_MODEL_CELERON_06        6
#define CPUID_MODEL_PM_09             9
#define CPUID_MODEL_PM_0D            13
#define CPUID_MODEL_PM_0E            14  // Yonah / Sossaman
#define CPUID_MODEL_CORE_0F          15  // Conroe / Merom
#define CPUID_MODEL_CORE_17        0x17  // Penryn
#define CPUID_MODEL_NEHALEM_1A     0x1a  // Nehalem / Gainestown
#define CPUID_MODEL_ATOM_1C        0x1c  // Silverthorne / Diamondville
#define CPUID_MODEL_CORE_1D        0x1d  // Dunnington
#define CPUID_MODEL_NEHALEM_1E     0x1e  // Lynnfield
#define CPUID_MODEL_NEHALEM_1F     0x1f  // Havendale
#define CPUID_MODEL_NEHALEM_25     0x25  // Westmere / Clarkdale
#define CPUID_MODEL_ATOM_26        0x26  // Lincroft
#define CPUID_MODEL_ATOM_27        0x27  // Saltwell
#define CPUID_MODEL_SANDYBRIDGE_2A 0x2a  // Sandybridge (desktop/mobile)
#define CPUID_MODEL_NEHALEM_2C     0x2c  // Westmere-EP
#define CPUID_MODEL_SANDYBRIDGE_2D 0x2d  // Sandybridge-EP
#define CPUID_MODEL_NEHALEM_2E     0x2e  // Nehalem-EX
#define CPUID_MODEL_NEHALEM_2F     0x2f  // Westmere-EX
#define CPUID_MODEL_ATOM_35        0x35  // Cloverview
#define CPUID_MODEL_ATOM_36        0x36  // Cedarview
#define CPUID_MODEL_ATOM_37        0x37  // Bay Trail
#define CPUID_MODEL_SANDYBRIDGE_3A 0x3a  // Ivy Bridge
#define CPUID_MODEL_HASWELL_3C     0x3c  // Haswell DT
#define CPUID_MODEL_BROADWELL_3D   0x3d  // Broadwell-Ult
#define CPUID_MODEL_SANDYBRIDGE_3E 0x3e  // Ivy Bridge-EP
#define CPUID_MODEL_HASWELL_3F     0x3f  // Haswell EP/EN/EX
#define CPUID_MODEL_HASWELL_45     0x45  // Haswell Ultrathin
#define CPUID_MODEL_HASWELL_46     0x46  // Haswell (Crystal Well)
#define CPUID_MODEL_BROADWELL_47   0x47  // Broadwell (Denlow)
#define CPUID_MODEL_ATOM_4A        0x4a  // Future Silvermont
#define CPUID_MODEL_ATOM_4C        0x4c  // Airmont
#define CPUID_MODEL_ATOM_4D        0x4d  // Avoton
#define CPUID_MODEL_SKYLAKE_4E     0x4e  // Skylake-Y / Kabylake U/Y ES
#define CPUID_MODEL_BROADWELL_4F   0x4f  // Broadwell EP/EN/EX
#define CPUID_MODEL_SKYLAKE_55     0x55  // Skylake EP/EN/EX
#define CPUID_MODEL_BROADWELL_56   0x56  // Broadwell DE
#define CPUID_MODEL_KNL_57         0x57  // Knights Landing
#define CPUID_MODEL_ATOM_5A        0x5a  // Future Silvermont
#define CPUID_MODEL_ATOM_5D        0x5d  // Future Silvermont
#define CPUID_MODEL_SKYLAKE_5E     0x5e  // Skylake-S / Kabylake S/H ES
#define CPUID_MODEL_ATOM_5F        0x5f  // Denverton
#define CPUID_MODEL_KNM_85         0x85  // Knights Mill
#define CPUID_MODEL_KABYLAKE_8E    0x8e  // Kabylake U/Y QS
#define CPUID_MODEL_KABYLAKE_9E    0x9e  // Kabylake S/H QS

/* Intel stepping information */
#define CPUID_STEPPING_KABYLAKE_ES     0x8  // Kabylake S/H/U/Y ES
#define CPUID_STEPPING_COFFEELAKE_A    0xA  // Coffeelake U/S/H
#define CPUID_STEPPING_COFFEELAKE_B    0xB  // Coffeelake S/H

#define CPUID_MODEL_PIII_07    7
#define CPUID_MODEL_PIII_08    8
#define CPUID_MODEL_PIII_0A    10

/* AMD model information */
#define CPUID_MODEL_BARCELONA_02      0x02 // Barcelona (Opteron & Phenom)
#define CPUID_MODEL_SHANGHAI_04       0x04 // Shanghai RB
#define CPUID_MODEL_SHANGHAI_05       0x05 // Shanghai BL
#define CPUID_MODEL_SHANGHAI_06       0x06 // Shanghai DA
#define CPUID_MODEL_ISTANBUL_MAGNY_08 0x08 // Istanbul (6 core) & Magny-cours (12) HY
#define CPUID_MODEL_ISTANBUL_MAGNY_09 0x09 // HY - G34 package
#define CPUID_MODEL_PHAROAH_HOUND_0A  0x0A // Pharoah Hound
#define CPUID_MODEL_PILEDRIVER_1F     0x1F // Max piledriver model defined in BKDG
#define CPUID_MODEL_PILEDRIVER_10     0x10 // family == CPUID_FAMILY_BULLDOZER
#define CPUID_MODEL_PILEDRIVER_02     0x02 // family == CPUID_FAMILY_BULLDOZER
#define CPUID_MODEL_OPTERON_REVF_41   0x41 // family == CPUID_FAMILY_K8
#define CPUID_MODEL_KYOTO_00          0x00 // family == CPUID_FAMILY_KYOTO
#define CPUID_MODEL_STEAMROLLER_3F    0x3F // Max Steamroller model defined in BKDG
#define CPUID_MODEL_STEAMROLLER_30    0x30 // family == CPUID_FAMILY_BULLDOZER
#define CPUID_MODEL_EXCAVATOR_60      0x60 // family == CPUID_FAMILY_BULLDOZER
#define CPUID_MODEL_EXCAVATOR_6F      0x6F // Max Excavator model defined in BKDG
#define CPUID_MODEL_ZEN_00            0x00 // family == CPUID_FAMILY_ZEN
#define CPUID_MODEL_ZEN_1F            0x1F // Max Zen model defined in BKDG

/* VIA model information */
#define CPUID_MODEL_NANO                15 // Isaiah

/* Hygon model information. */
#define CPUID_MODEL_DHYANA_A             0 // Dhyana A

/*
 *----------------------------------------------------------------------
 *
 * CPUID_IsVendor{AMD,Intel,VIA,Hygon} --
 *
 *      Determines if the vendor string in cpuid id0 is from
 *      {AMD,Intel,VIA,Hygon}.
 *
 * Results:
 *      True iff vendor string is CPUID_{AMD,INTEL,VIA,HYGON}_VENDOR_STRING
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */
static INLINE Bool
CPUID_IsRawVendor(CPUIDRegs *id0, const char* vendor)
{
   // hard to get strcmp() in some environments, so do it in the raw
   return (id0->ebx == *(const uint32 *) (vendor + 0) &&
           id0->ecx == *(const uint32 *) (vendor + 4) &&
           id0->edx == *(const uint32 *) (vendor + 8));
}

static INLINE Bool
CPUID_IsVendorAMD(CPUIDRegs *id0)
{
   return CPUID_IsRawVendor(id0, CPUID_AMD_VENDOR_STRING);
}

static INLINE Bool
CPUID_IsVendorIntel(CPUIDRegs *id0)
{
   return CPUID_IsRawVendor(id0, CPUID_INTEL_VENDOR_STRING);
}

static INLINE Bool
CPUID_IsVendorVIA(CPUIDRegs *id0)
{
   return CPUID_IsRawVendor(id0, CPUID_VIA_VENDOR_STRING);
}

static INLINE Bool
CPUID_IsVendorHygon(CPUIDRegs *id0)
{
   return CPUID_IsRawVendor(id0, CPUID_HYGON_VENDOR_STRING);
}

static INLINE uint32
CPUID_EFFECTIVE_FAMILY(uint32 v) /* %eax from CPUID with %eax=1. */
{
   uint32 f = CPUID_GET(1, EAX, FAMILY, v);
   return f != CPUID_FAMILY_EXTENDED ? f : f +
      CPUID_GET(1, EAX, EXTENDED_FAMILY, v);
}

/* Normally only used when FAMILY==CPUID_FAMILY_EXTENDED, but Intel is
 * now using the extended model field for FAMILY==CPUID_FAMILY_P6 to
 * refer to the newer Core2 CPUs
 */
static INLINE uint32
CPUID_EFFECTIVE_MODEL(uint32 v) /* %eax from CPUID with %eax=1. */
{
   uint32 m = CPUID_GET(1, EAX, MODEL, v);
   uint32 em = CPUID_GET(1, EAX, EXTENDED_MODEL, v);
   return m + (em << 4);
}

static INLINE uint32
CPUID_EFFECTIVE_STEPPING(uint32 v) /* %eax from CPUID with %eax=1. */
{
   return CPUID_GET(1, EAX, STEPPING, v);
}

/*
 * Notice that CPUID families for Intel and AMD overlap. The following macros
 * should only be used AFTER the manufacturer has been established (through
 * the use of CPUID standard function 0).
 */
static INLINE Bool
CPUID_FAMILY_IS_486(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_486;
}

static INLINE Bool
CPUID_FAMILY_IS_P5(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_P5;
}

static INLINE Bool
CPUID_FAMILY_IS_P6(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_P6;
}

static INLINE Bool
CPUID_FAMILY_IS_PENTIUM4(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_P4;
}

/*
 * Intel Pentium M processors are Yonah/Sossaman or an older P-M
 */
static INLINE Bool
CPUID_UARCH_IS_PENTIUM_M(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_PM_09 ||
           CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_PM_0D ||
           CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_PM_0E);
}

/*
 * Intel Core processors are Merom, Conroe, Woodcrest, Clovertown,
 * Penryn, Dunnington, Kentsfield, Yorktown, Harpertown, ........
 */
static INLINE Bool
CPUID_UARCH_IS_CORE(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   uint32 model = CPUID_EFFECTIVE_MODEL(v);
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          model >= CPUID_MODEL_CORE_0F &&
          (model < CPUID_MODEL_NEHALEM_1A ||
           model == CPUID_MODEL_CORE_1D);
}

/*
 * Intel Nehalem processors are: Nehalem, Gainestown, Lynnfield, Clarkdale.
 */
static INLINE Bool
CPUID_UARCH_IS_NEHALEM(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);

return CPUID_FAMILY_IS_P6(v) &&
          (effectiveModel == CPUID_MODEL_NEHALEM_1A ||
           effectiveModel == CPUID_MODEL_NEHALEM_1E ||
           effectiveModel == CPUID_MODEL_NEHALEM_1F ||
           effectiveModel == CPUID_MODEL_NEHALEM_25 ||
           effectiveModel == CPUID_MODEL_NEHALEM_2C ||
           effectiveModel == CPUID_MODEL_NEHALEM_2E ||
           effectiveModel == CPUID_MODEL_NEHALEM_2F);
}

static INLINE Bool
CPUID_UARCH_IS_SANDYBRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);

return CPUID_FAMILY_IS_P6(v) &&
          (effectiveModel == CPUID_MODEL_SANDYBRIDGE_2A ||
           effectiveModel == CPUID_MODEL_SANDYBRIDGE_2D ||
           effectiveModel == CPUID_MODEL_SANDYBRIDGE_3E ||
           effectiveModel == CPUID_MODEL_SANDYBRIDGE_3A);
}

static INLINE Bool
CPUID_MODEL_IS_BROADWELL(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);

return CPUID_FAMILY_IS_P6(v) &&
          (effectiveModel == CPUID_MODEL_BROADWELL_3D ||
           effectiveModel == CPUID_MODEL_BROADWELL_47 ||
           effectiveModel == CPUID_MODEL_BROADWELL_4F ||
           effectiveModel == CPUID_MODEL_BROADWELL_56);
}

static INLINE Bool
CPUID_MODEL_IS_HASWELL(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);

return CPUID_FAMILY_IS_P6(v) &&
          (effectiveModel == CPUID_MODEL_HASWELL_3C ||
           effectiveModel == CPUID_MODEL_HASWELL_3F ||
           effectiveModel == CPUID_MODEL_HASWELL_45 ||
           effectiveModel == CPUID_MODEL_HASWELL_46);
}

static INLINE Bool
CPUID_MODEL_IS_SKYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          ((CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_5E &&
            CPUID_EFFECTIVE_STEPPING(v) != CPUID_STEPPING_KABYLAKE_ES) ||
            CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_55 ||
           (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_4E &&
            CPUID_EFFECTIVE_STEPPING(v) != CPUID_STEPPING_KABYLAKE_ES));
}

static INLINE Bool
CPUID_MODEL_IS_COFFEELAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          ((CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KABYLAKE_9E           &&
            (CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_COFFEELAKE_A   ||
             CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_COFFEELAKE_B)) ||
           (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KABYLAKE_8E           &&
            CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_COFFEELAKE_A));
}

static INLINE Bool
CPUID_MODEL_IS_KABYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          !CPUID_MODEL_IS_COFFEELAKE(v) &&
          (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KABYLAKE_9E         ||
           CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KABYLAKE_8E         ||
           (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_5E         &&
            CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_KABYLAKE_ES) ||
           (CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_SKYLAKE_4E         &&
            CPUID_EFFECTIVE_STEPPING(v) == CPUID_STEPPING_KABYLAKE_ES));
}

static INLINE Bool
CPUID_UARCH_IS_SKYLAKE(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          (CPUID_MODEL_IS_COFFEELAKE(v) ||
           CPUID_MODEL_IS_KABYLAKE(v)   ||
           CPUID_MODEL_IS_SKYLAKE(v));
}

static INLINE Bool
CPUID_UARCH_IS_HASWELL(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          (CPUID_MODEL_IS_BROADWELL(v) || CPUID_MODEL_IS_HASWELL(v));
}

static INLINE Bool
CPUID_MODEL_IS_CENTERTON(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_1C;
}

static INLINE Bool
CPUID_MODEL_IS_AVOTON(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_4D;
}

static INLINE Bool
CPUID_MODEL_IS_BAYTRAIL(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_37;
}

static INLINE Bool
CPUID_UARCH_IS_SILVERMONT(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          (CPUID_MODEL_IS_AVOTON(v) || CPUID_MODEL_IS_BAYTRAIL(v));
}

static INLINE Bool
CPUID_MODEL_IS_DENVERTON(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_ATOM_5F;
}

static INLINE Bool
CPUID_MODEL_IS_WESTMERE(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);

return CPUID_FAMILY_IS_P6(v) &&
          (effectiveModel == CPUID_MODEL_NEHALEM_25 || // Clarkdale
           effectiveModel == CPUID_MODEL_NEHALEM_2C || // Westmere-EP
           effectiveModel == CPUID_MODEL_NEHALEM_2F);  // Westmere-EX
}

static INLINE Bool
CPUID_MODEL_IS_SANDYBRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);

return CPUID_FAMILY_IS_P6(v) &&
          (effectiveModel == CPUID_MODEL_SANDYBRIDGE_2A ||
           effectiveModel == CPUID_MODEL_SANDYBRIDGE_2D);
}

static INLINE Bool
CPUID_MODEL_IS_IVYBRIDGE(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   uint32 effectiveModel = CPUID_EFFECTIVE_MODEL(v);

return CPUID_FAMILY_IS_P6(v) && (
       effectiveModel == CPUID_MODEL_SANDYBRIDGE_3E ||
       effectiveModel == CPUID_MODEL_SANDYBRIDGE_3A);
}

static INLINE Bool
CPUID_MODEL_IS_KNIGHTS_LANDING(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KNL_57;
}

static INLINE Bool
CPUID_MODEL_IS_KNIGHTS_MILL(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is Intel. */
   return CPUID_FAMILY_IS_P6(v) &&
          CPUID_EFFECTIVE_MODEL(v) == CPUID_MODEL_KNM_85;
}

static INLINE Bool
CPUID_FAMILY_IS_K7(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_K7;
}

static INLINE Bool
CPUID_FAMILY_IS_K8(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_K8;
}

/*
 *----------------------------------------------------------------------
 *
 * CPUID_FAMILY_IS_K8EXT --
 *
 *      Return TRUE for family K8 with effective model >= 0x10.
 *
 *----------------------------------------------------------------------
 */
static INLINE Bool
CPUID_FAMILY_IS_K8EXT(uint32 eax)
{
   return CPUID_FAMILY_IS_K8(eax) &&
          CPUID_GET(1, EAX, EXTENDED_MODEL, eax) != 0;
}

static INLINE Bool
CPUID_FAMILY_IS_K8L(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_K8L ||
          CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_LLANO;
}

static INLINE Bool
CPUID_FAMILY_IS_LLANO(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_LLANO;
}

static INLINE Bool
CPUID_FAMILY_IS_K8MOBILE(uint32 eax)
{
   /* Essentially a K8 (not K8L) part, but with mobile features. */
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_K8MOBILE;
}

static INLINE Bool
CPUID_FAMILY_IS_K8STAR(uint32 eax)
{
   /*
    * Read function name as "K8*", as in wildcard.
    * Matches K8 or K8L or K8MOBILE
    */
   return CPUID_FAMILY_IS_K8(eax) || CPUID_FAMILY_IS_K8L(eax) ||
          CPUID_FAMILY_IS_K8MOBILE(eax);
}

static INLINE Bool
CPUID_FAMILY_IS_BOBCAT(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BOBCAT;
}

static INLINE Bool
CPUID_FAMILY_IS_BULLDOZER(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER;
}

static INLINE Bool
CPUID_FAMILY_IS_KYOTO(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_KYOTO;
}

static INLINE Bool
CPUID_FAMILY_IS_ZEN(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN;
}

/*
 * AMD Barcelona (of either Opteron or Phenom kind).
 */
static INLINE Bool
CPUID_MODEL_IS_BARCELONA(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is AMD. */
   return CPUID_EFFECTIVE_FAMILY(v) == CPUID_FAMILY_K8L &&
          CPUID_EFFECTIVE_MODEL(v)  == CPUID_MODEL_BARCELONA_02;
}

static INLINE Bool
CPUID_MODEL_IS_SHANGHAI(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is AMD. */
   return CPUID_EFFECTIVE_FAMILY(v) == CPUID_FAMILY_K8L &&
          (CPUID_MODEL_SHANGHAI_04  <= CPUID_EFFECTIVE_MODEL(v) &&
           CPUID_EFFECTIVE_MODEL(v) <= CPUID_MODEL_SHANGHAI_06);
}

static INLINE Bool
CPUID_MODEL_IS_ISTANBUL_MAGNY(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is AMD. */
   return CPUID_EFFECTIVE_FAMILY(v) == CPUID_FAMILY_K8L &&
          (CPUID_MODEL_ISTANBUL_MAGNY_08 <= CPUID_EFFECTIVE_MODEL(v) &&
           CPUID_EFFECTIVE_MODEL(v)      <= CPUID_MODEL_ISTANBUL_MAGNY_09);
}

static INLINE Bool
CPUID_MODEL_IS_PHAROAH_HOUND(uint32 v) // IN: %eax from CPUID with %eax=1.
{
   /* Assumes the CPU manufacturer is AMD. */
   return CPUID_EFFECTIVE_FAMILY(v) == CPUID_FAMILY_K8L &&
          CPUID_EFFECTIVE_MODEL(v)  == CPUID_MODEL_PHAROAH_HOUND_0A;
}

static INLINE Bool
CPUID_MODEL_IS_BULLDOZER(uint32 eax)
{
   /*
    * Bulldozer is models of family 0x15 that are below 10 excluding
    * Piledriver 02.
    */
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER &&
          CPUID_EFFECTIVE_MODEL(eax)  < CPUID_MODEL_PILEDRIVER_10 &&
          CPUID_EFFECTIVE_MODEL(eax) != CPUID_MODEL_PILEDRIVER_02;
}

static INLINE Bool
CPUID_MODEL_IS_PILEDRIVER(uint32 eax)
{
   /* Piledriver is models 0x02 & 0x10 of family 0x15 (so far). */
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER &&
          ((CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_PILEDRIVER_10 &&
            CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_PILEDRIVER_1F) ||
           CPUID_EFFECTIVE_MODEL(eax) == CPUID_MODEL_PILEDRIVER_02);
}

static INLINE Bool
CPUID_MODEL_IS_STEAMROLLER(uint32 eax)
{
   /* Steamroller is model 0x30 of family 0x15 (so far). */
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER &&
          (CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_STEAMROLLER_30 &&
           CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_STEAMROLLER_3F);
}

static INLINE Bool
CPUID_MODEL_IS_EXCAVATOR(uint32 eax)
{
   /* Excavator is model 0x60 of family 0x15 (so far). */
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_BULLDOZER &&
          (CPUID_EFFECTIVE_MODEL(eax) >= CPUID_MODEL_EXCAVATOR_60 &&
           CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_EXCAVATOR_6F);
}

static INLINE Bool
CPUID_MODEL_IS_KYOTO(uint32 eax)
{
   /* Kyoto is models 0x00 of family 0x16 (so far). */
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_KYOTO &&
          CPUID_EFFECTIVE_MODEL(eax) == CPUID_MODEL_KYOTO_00;
}

static INLINE Bool
CPUID_MODEL_IS_ZEN(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_ZEN &&
          CPUID_EFFECTIVE_MODEL(eax) <= CPUID_MODEL_ZEN_1F;
}

static INLINE Bool
CPUID_FAMILY_IS_DHYANA(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_DHYANA;
}

static INLINE Bool
CPUID_MODEL_IS_DHYANA_A(uint32 eax)
{
   return CPUID_EFFECTIVE_FAMILY(eax) == CPUID_FAMILY_DHYANA &&
          CPUID_EFFECTIVE_MODEL(eax)  == CPUID_MODEL_DHYANA_A;
}

#define CPUID_TYPE_PRIMARY     0
#define CPUID_TYPE_OVERDRIVE   1
#define CPUID_TYPE_SECONDARY   2

#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_NULL      0
#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_DATA      1
#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_INST      2
#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_TYPE_UNIF      3
#define CPUID_LEAF4_CACHE_TYPE_NULL      0
#define CPUID_LEAF4_CACHE_TYPE_DATA      1
#define CPUID_LEAF4_CACHE_TYPE_INST      2
#define CPUID_LEAF4_CACHE_TYPE_UNIF      3
#define CPUID_LEAF4_CACHE_INDEXING_DIRECT  0
#define CPUID_LEAF4_CACHE_INDEXING_COMPLEX 1

#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_SELF_INIT      0x00000100
#define CPUID_INTEL_ID4EAX_LEAF4_CACHE_FULLY_ASSOC    0x00000200
#define CPUID_LEAF4_CACHE_SELF_INIT      0x00000100
#define CPUID_LEAF4_CACHE_FULLY_ASSOC    0x00000200

#define CPUID_INTEL_IDBECX_LEVEL_TYPE_INVALID   0
#define CPUID_INTEL_IDBECX_LEVEL_TYPE_SMT       1
#define CPUID_INTEL_IDBECX_LEVEL_TYPE_CORE      2
#define CPUID_TOPOLOGY_LEVEL_TYPE_INVALID   0
#define CPUID_TOPOLOGY_LEVEL_TYPE_SMT       1
#define CPUID_TOPOLOGY_LEVEL_TYPE_CORE      2

/*
 * For certain AMD processors, an lfence instruction is necessary at various
 * places to ensure ordering.
 */

static INLINE Bool
CPUID_VendorRequiresFence(CpuidVendor vendor)
{
   return vendor == CPUID_VENDOR_AMD;
}

static INLINE Bool
CPUID_VersionRequiresFence(uint32 version)
{
   return CPUID_EFFECTIVE_FAMILY(version) == CPUID_FAMILY_K8 &&
          CPUID_EFFECTIVE_MODEL(version) < 0x40;
}

static INLINE Bool
CPUID_ID0RequiresFence(CPUIDRegs *id0)
{
   if (id0->eax == 0) {
      return FALSE;
   }
   return CPUID_IsVendorAMD(id0);
}

static INLINE Bool
CPUID_ID1RequiresFence(CPUIDRegs *id1)
{
   return CPUID_VersionRequiresFence(id1->eax);
}

static INLINE Bool
CPUID_RequiresFence(CpuidVendor vendor, // IN
                    uint32 version)      // IN: %eax from CPUID with %eax=1.
{
   return CPUID_VendorRequiresFence(vendor) &&
          CPUID_VersionRequiresFence(version);
}

/*
 * The following low-level functions compute the number of
 * cores per cpu.  They should be used cautiously because
 * they do not necessarily work on all types of CPUs.
 * High-level functions that are correct for all CPUs are
 * available elsewhere: see lib/cpuidInfo/cpuidInfo.c.
 */

static INLINE uint32
CPUID_IntelCoresPerPackage(uint32 v) /* %eax from CPUID with %eax=4 and %ecx=0. */
{
   // Note: This is not guaranteed to work on older Intel CPUs.
   return 1 + CPUID_GET(4, EAX, LEAF4_CORE_COUNT, v);
}

static INLINE uint32
CPUID_AMDCoresPerPackage(uint32 v) /* %ecx from CPUID with %eax=0x80000008. */
{
   // Note: This is not guaranteed to work on older AMD CPUs.
   return 1 + CPUID_GET(0x80000008, ECX, LEAF88_CORE_COUNT, v);
}

/*
 * Hypervisor CPUID space is 0x400000XX.
 */
static INLINE Bool
CPUID_IsHypervisorLevel(uint32 level)
{
   return (level & 0xffffff00) == 0x40000000;
}

/*
 *----------------------------------------------------------------------
 *
 * CPUID_LevelUsesEcx --
 *
 *      Returns TRUE for leaves that support input ECX != 0 (subleaves).
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
CPUID_LevelUsesEcx(uint32 level) {
   switch (level)
   {

#define CPUIDLEVEL(t, s, v, c, h)   \
      case v:                       \
         return c != 0;

CPUID_ALL_LEVELS

#undef CPUIDLEVEL

default:
         return FALSE;
   }
}

/*
 *----------------------------------------------------------------------
 *
 * CPUID_IsValid*Subleaf --
 *
 *      Functions to determine the last subleaf for the level specified
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
CPUID_IsValidBSubleaf(uint32 ebx)  // IN: %ebx = cpuid.b.sublevel.ebx
{
   return ebx != 0;
}

static INLINE Bool
CPUID_IsValid4Subleaf(uint32 eax)  // IN: %eax = cpuid.4.sublevel.eax
{
   return eax != 0;
}

static INLINE Bool
CPUID_IsValid7Subleaf(uint32 eax, uint32 subleaf)  // IN: %eax = cpuid.7.0.eax
{
   /*
    * cpuid.7.0.eax is the max ecx (subleaf) index
    */
   return subleaf <= eax;
}

/*
 *----------------------------------------------------------------------
 *
 * CPUID_IsValidDSubleaf --
 *
 *    It is the caller's repsonsibility to determine if the processor
 *    supports XSAVE and therefore has D sub-leaves.
 *
 *----------------------------------------------------------------------
 */
static INLINE Bool
CPUID_IsValidDSubleaf(uint32 subleaf)  // IN: subleaf to check
{
   return subleaf <= 63;
}

/*
 *----------------------------------------------------------------------
 *
 * CPUID_SupportsMsrPlatformInfo --
 *
 *    Uses vendor and cpuid.1.0.eax to determine if the processor
 *    supports MSR_PLATFORM_INFO.
 *
 *----------------------------------------------------------------------
 */
static INLINE Bool
CPUID_SupportsMsrPlatformInfo(CpuidVendor vendor, uint32 version)
{
   return vendor == CPUID_VENDOR_INTEL &&
          (CPUID_UARCH_IS_NEHALEM(version)     ||
           CPUID_UARCH_IS_SANDYBRIDGE(version) ||
           CPUID_UARCH_IS_HASWELL(version)     ||
           CPUID_UARCH_IS_SKYLAKE(version)     ||
           CPUID_MODEL_IS_KNIGHTS_LANDING(version) ||
           CPUID_MODEL_IS_DENVERTON(version) ||
           CPUID_UARCH_IS_SILVERMONT(version));
}

#ifdef _MSC_VER
#pragma warning (pop)
#endif

#if defined __cplusplus
} // extern "C"
#endif

#endif // _X86CPUID_H_
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vm_basic_asm.h������������������������������������������������������������������0000444�0000000�0000000�00000071321�13432725350�016207� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2003-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vm_basic_asm.h
 *
 *	Basic asm macros
 */

#ifndef _VM_BASIC_ASM_H_
#define _VM_BASIC_ASM_H_

#define INCLUDE_ALLOW_USERLEVEL

#include "vm_basic_types.h"

#if defined VM_X86_64
#include "vm_basic_asm_x86_common.h"
#include "vm_basic_asm_x86_64.h"
#elif defined VM_X86_32
#include "vm_basic_asm_x86_common.h"
#include "vm_basic_asm_x86.h"
#elif defined VM_ARM_32
#include "vm_basic_asm_arm32.h"
#define MUL64_NO_ASM 1
#include "mul64.h"
#elif defined VM_ARM_64
#include "arm64_basic_defs.h"
#include "vm_basic_asm_arm64.h"
#else
#define MUL64_NO_ASM 1
#include "mul64.h"
#endif

#if defined __cplusplus
extern "C" {
#endif

/*
 * Locate most and least significant bit set functions. Use our own name
 * space to avoid namespace collisions. The new names follow a pattern,
 * <prefix><size><option>, where:
 *
 * <prefix> is [lm]ssb (least/most significant bit set)
 * <size> is size of the argument: 32 (32-bit), 64 (64-bit) or Ptr (pointer)
 * <option> is for alternative versions of the functions
 *
 * NAME        FUNCTION                    BITS     FUNC(0)
 *-----        --------                    ----     -------
 * lssb32_0    LSB set (uint32)            0..31    -1
 * mssb32_0    MSB set (uint32)            0..31    -1
 * lssb64_0    LSB set (uint64)            0..63    -1
 * mssb64_0    MSB set (uint64)            0..63    -1
 * lssbPtr_0   LSB set (uintptr_t;32-bit)  0..31    -1
 * lssbPtr_0   LSB set (uintptr_t;64-bit)  0..63    -1
 * mssbPtr_0   MSB set (uintptr_t;32-bit)  0..31    -1
 * mssbPtr_0   MSB set (uintptr_t;64-bit)  0..63    -1
 * lssbPtr     LSB set (uintptr_t;32-bit)  1..32    0
 * lssbPtr     LSB set (uintptr_t;64-bit)  1..64    0
 * mssbPtr     MSB set (uintptr_t;32-bit)  1..32    0
 * mssbPtr     MSB set (uintptr_t;64-bit)  1..64    0
 * lssb32      LSB set (uint32)            1..32    0
 * mssb32      MSB set (uint32)            1..32    0
 * lssb64      LSB set (uint64)            1..64    0
 * mssb64      MSB set (uint64)            1..64    0
 */

#ifdef _MSC_VER
static INLINE int
lssb32_0(const uint32 value)
{
   unsigned long idx;
   unsigned char ret;

if (UNLIKELY(value == 0)) {
      return -1;
   }
   ret = _BitScanForward(&idx, (unsigned long)value);
#ifdef __analysis_assume
   __analysis_assume(ret != 0);
#endif

#pragma warning(suppress: 6001 6102) // Suppress bogus complaint that idx may be uninitialized in error case
   return idx;
}

static INLINE int
mssb32_0(const uint32 value)
{
   unsigned long idx;
   unsigned char ret;

if (UNLIKELY(value == 0)) {
      return -1;
   }
   ret = _BitScanReverse(&idx, (unsigned long)value);
#ifdef __analysis_assume
   __analysis_assume(ret != 0);
#endif

#pragma warning(suppress: 6001 6102) // Suppress bogus complaint that idx may be uninitialized in error case
   return idx;
}

static INLINE int
lssb64_0(const uint64 value)
{
   if (UNLIKELY(value == 0)) {
      return -1;
   } else {
#ifdef VM_X86_64
      unsigned long idx;
      unsigned char ret;

ret = _BitScanForward64(&idx, (unsigned __int64)value);
#ifdef __analysis_assume
      __analysis_assume(ret != 0);
#endif

#pragma warning(suppress: 6001 6102) // Suppress bogus complaint that idx may be uninitialized in error case
      return idx;
#else
      /* The coding was chosen to minimize conditionals and operations */
      int lowFirstBit = lssb32_0((uint32) value);
      if (lowFirstBit == -1) {
         lowFirstBit = lssb32_0((uint32) (value >> 32));
         if (lowFirstBit != -1) {
            return lowFirstBit + 32;
         }
      }
      return lowFirstBit;
#endif
   }
}

static INLINE int
mssb64_0(const uint64 value)
{
   if (UNLIKELY(value == 0)) {
      return -1;
   } else {
#ifdef VM_X86_64
      unsigned long idx;
      unsigned char ret;

ret = _BitScanReverse64(&idx, (unsigned __int64)value);
#ifdef __analysis_assume
      __analysis_assume(ret != 0);
#endif

#pragma warning(suppress: 6001 6102) // Suppress bogus complaint that idx may be uninitialized in error case
      return idx;
#else
      /* The coding was chosen to minimize conditionals and operations */
      if (value > 0xFFFFFFFFULL) {
         return 32 + mssb32_0((uint32) (value >> 32));
      }
      return mssb32_0((uint32) value);
#endif
   }
}
#endif

#ifdef __GNUC__

#ifdef VM_X86_ANY
#define USE_ARCH_X86_CUSTOM
#endif

/* **********************************************************
 *  GCC's intrinsics for the lssb and mssb family produce sub-optimal code,
 *  so we use inline assembly to improve matters.  However, GCC cannot
 *  propagate constants through inline assembly, so we help GCC out by
 *  allowing it to use its intrinsics for compile-time constant values.
 *  Some day, GCC will make better code and these can collapse to intrinsics.
 *
 *  For example, in Decoder_AddressSize, inlined into VVT_GetVTInstrInfo:
 *  __builtin_ffs(a) compiles to:
 *  mov   $0xffffffff, %esi
 *  bsf   %eax, %eax
 *  cmovz %esi, %eax
 *  sub   $0x1, %eax
 *  and   $0x7, %eax
 *
 *  While the code below compiles to:
 *  bsf   %eax, %eax
 *  sub   $0x1, %eax
 *
 *  Ideally, GCC should have recognized non-zero input in the first case.
 *  Other instances of the intrinsic produce code like
 *  sub $1, %eax; add $1, %eax; clts
 * **********************************************************
 */

#if __GNUC__ < 4
#define FEWER_BUILTINS
#endif

static INLINE int
lssb32_0(uint32 value)
{
#ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
         return -1;
      } else {
         int pos;
         __asm__ ("bsfl %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
         return pos;
      }
   }
#endif
   return __builtin_ffs(value) - 1;
}

#ifndef FEWER_BUILTINS
static INLINE int
mssb32_0(uint32 value)
{
   /*
    * We must keep the UNLIKELY(...) outside the #if defined ...
    * because __builtin_clz(0) is undefined according to gcc's
    * documentation.
    */
   if (UNLIKELY(value == 0)) {
      return -1;
   } else {
      int pos;
#ifdef USE_ARCH_X86_CUSTOM
      if (!__builtin_constant_p(value)) {
         __asm__ ("bsrl %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
         return pos;
      }
#endif
      pos = 32 - __builtin_clz(value) - 1;
      return pos;
   }
}

static INLINE int
lssb64_0(const uint64 value)
{
#ifdef USE_ARCH_X86_CUSTOM
   if (!__builtin_constant_p(value)) {
      if (UNLIKELY(value == 0)) {
         return -1;
      } else {
         intptr_t pos;
#ifdef VM_X86_64
         __asm__ ("bsf %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
#else
         /* The coding was chosen to minimize conditionals and operations */
         pos = lssb32_0((uint32) value);
         if (pos == -1) {
            pos = lssb32_0((uint32) (value >> 32));
            if (pos != -1) {
               return pos + 32;
            }
         }
#endif
         return pos;
      }
   }
#endif
   return __builtin_ffsll(value) - 1;
}
#endif /* !FEWER_BUILTINS */

#ifdef FEWER_BUILTINS
/* GCC 3.3.x does not like __bulitin_clz or __builtin_ffsll. */
static INLINE int
mssb32_0(uint32 value)
{
   if (UNLIKELY(value == 0)) {
      return -1;
   } else {
      int pos;
      __asm__ __volatile__("bsrl %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
      return pos;
   }
}

static INLINE int
lssb64_0(const uint64 value)
{
   if (UNLIKELY(value == 0)) {
      return -1;
   } else {
      intptr_t pos;

#ifdef VM_X86_64
      __asm__ __volatile__("bsf %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
#else
      /* The coding was chosen to minimize conditionals and operations */
      pos = lssb32_0((uint32) value);
      if (pos == -1) {
         pos = lssb32_0((uint32) (value >> 32));
         if (pos != -1) {
            return pos + 32;
         }
      }
#endif /* VM_X86_64 */
      return pos;
   }
}
#endif /* FEWER_BUILTINS */

static INLINE int
mssb64_0(const uint64 value)
{
   if (UNLIKELY(value == 0)) {
      return -1;
   } else {
      intptr_t pos;

#ifdef USE_ARCH_X86_CUSTOM
#ifdef VM_X86_64
      __asm__ ("bsr %1, %0\n" : "=r" (pos) : "rm" (value) : "cc");
#else
      /* The coding was chosen to minimize conditionals and operations */
      if (value > 0xFFFFFFFFULL) {
         pos = 32 + mssb32_0((uint32) (value >> 32));
      } else {
         pos = mssb32_0((uint32) value);
      }
#endif
#else
      pos = 64 - __builtin_clzll(value) - 1;
#endif

return pos;
   }
}

#ifdef USE_ARCH_X86_CUSTOM
#undef USE_ARCH_X86_CUSTOM
#endif

#endif // __GNUC__

static INLINE int
lssbPtr_0(const uintptr_t value)
{
#ifdef VM_64BIT
   return lssb64_0((uint64) value);
#else
   return lssb32_0((uint32) value);
#endif
}

static INLINE int
lssbPtr(const uintptr_t value)
{
   return lssbPtr_0(value) + 1;
}

static INLINE int
mssbPtr_0(const uintptr_t value)
{
#ifdef VM_64BIT
   return mssb64_0((uint64) value);
#else
   return mssb32_0((uint32) value);
#endif
}

static INLINE int
mssbPtr(const uintptr_t value)
{
   return mssbPtr_0(value) + 1;
}

static INLINE int
lssb32(const uint32 value)
{
   return lssb32_0(value) + 1;
}

static INLINE int
mssb32(const uint32 value)
{
   return mssb32_0(value) + 1;
}

static INLINE int
lssb64(const uint64 value)
{
   return lssb64_0(value) + 1;
}

static INLINE int
mssb64(const uint64 value)
{
   return mssb64_0(value) + 1;
}

#ifdef __GNUC__
#if defined(VM_X86_ANY) || defined(VM_ARM_ANY)

/*
 *----------------------------------------------------------------------
 *
 * uint16set --
 *
 *      memset a given address with an uint16 value, count times.
 *
 * Results:
 *      Pointer to filled memory range.
 *
 * Side effects:
 *      As with memset.
 *
 *----------------------------------------------------------------------
 */

static INLINE void *
uint16set(void *dst, uint16 val, size_t count)
{
#ifdef VM_ARM_32
   void *tmpDst = dst;

__asm__ __volatile__ (
      "cmp     %1, #0\n\t"
      "beq     2f\n"
      "1:\n\t"
      "strh    %2, [%0], #2\n\t"
      "subs    %1, %1, #1\n\t"
      "bne     1b\n"
      "2:"
      : "+r" (tmpDst), "+r" (count)
      : "r" (val)
      : "cc", "memory");
#elif defined(VM_ARM_64)
   void   *tmpDst = dst;
   uint64  tmpVal = 0;

if (count == 0) {
      return dst;
   }

__asm__ __volatile__ (
      "cbz     %3, 1f\n\t"

// Copy 16 bits twice...
      "bfm     %2, %3, #0, #15\n\t"
      "lsl     %2, %2, #16\n\t"
      "bfm     %2, %3, #0, #15\n\t"

// Copy 32 bits from the bottom of the reg. to the top...
      "lsl     %2, %2, #32\n\t"
      "bfm     %2, %2, #32, #63\n"

// Copy into dst 8 bytes (4 uint16s) at a time
      "1:\t"
      "cmp     %1, #4\n\t"
      "b.lo    2f\n\t"
      "str     %2, [%0], #8\n\t"
      "sub     %1, %1, #4\n\t"
      "b       1b\n"

// Copy into dst 4 bytes at a time
      "2:\t"
      "cmp     %1, #2\n\t"
      "b.lo    3f\n\t"
      "str     %w2, [%0], #4\n\t"
      "sub     %1, %1, #2\n\t"
      "b       2b\n"

// We have 1 or zero items left...
      "3:\t"
      "cbz     %1, 4f\n\t"
      "strh    %w2, [%0]\n"
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (tmpVal)
      : "r" (val)
      : "cc", "memory");
#else
   size_t dummy0;
   void *dummy1;

__asm__ __volatile__("\t"
                        "cld"            "\n\t"
                        "rep ; stosw"    "\n"
                        : "=c" (dummy0), "=D" (dummy1)
                        : "0" (count), "1" (dst), "a" (val)
                        : "memory", "cc"
      );
#endif
   return dst;
}

/*
 *----------------------------------------------------------------------
 *
 * uint32set --
 *
 *      memset a given address with an uint32 value, count times.
 *
 * Results:
 *      Pointer to filled memory range.
 *
 * Side effects:
 *      As with memset.
 *
 *----------------------------------------------------------------------
 */

static INLINE void *
uint32set(void *dst, uint32 val, size_t count)
{
#ifdef VM_ARM_32
   void *tmpDst = dst;

__asm__ __volatile__ (
      "cmp     %1, #0\n\t"
      "beq     2f\n"
      "1:\n\t"
      "str     %2, [%0], #4\n\t"
      "subs    %1, %1, #1\n\t"
      "bne     1b\n"
      "2:"
      : "+r" (tmpDst), "+r" (count)
      : "r" (val)
      : "cc", "memory");
#elif defined(VM_ARM_64)
   void   *tmpDst = dst;

if (count == 0) {
      return dst;
   }

__asm__ __volatile__ (
      "cbz     %2, 1f\n\t"

// Drop our value in the top 32 bits, then copy from there to the bottom
      "lsl     %2, %2, #32\n\t"
      "bfm     %2, %2, #32, #63\n"

// Copy four at a time
      "1:\t"
      "cmp     %1, #16\n\t"
      "b.lo    2f\n\t"
      "stp     %2, %2, [%0], #16\n\t"
      "stp     %2, %2, [%0], #16\n\t"
      "stp     %2, %2, [%0], #16\n\t"
      "stp     %2, %2, [%0], #16\n\t"
      "sub     %1, %1, #16\n\t"
      "b       1b\n"

// Copy remaining pairs of data
      "2:\t"
      "cmp     %1, #2\n\t"
      "b.lo    3f\n\t"
      "str     %2, [%0], #8\n\t"
      "sub     %1, %1, #2\n\t"
      "b       2b\n"

// One or zero values left to copy
      "3:\t"
      "cbz     %1, 4f\n\t"
      "str     %w2, [%0]\n\t" // No incr
      "4:"
      : "+r" (tmpDst), "+r" (count), "+r" (val)
      :
      : "cc", "memory");
#else
   size_t dummy0;
   void *dummy1;

__asm__ __volatile__("\t"
                        "cld"            "\n\t"
                        "rep ; stosl"    "\n"
                        : "=c" (dummy0), "=D" (dummy1)
                        : "0" (count), "1" (dst), "a" (val)
                        : "memory", "cc"
      );
#endif
   return dst;
}

#else /* unknown system: rely on C to write */
static INLINE void *
uint16set(void *dst, uint16 val, size_t count)
{
   size_t i;
   for (i = 0; i < count; i++) {
     ((uint16 *) dst)[i] = val;
   }
   return dst;
}

static INLINE void *
uint32set(void *dst, uint32 val, size_t count)
{
   size_t i;
   for (i = 0; i < count; i++) {
     ((uint32 *) dst)[i] = val;
   }
   return dst;
}
#endif // defined(VM_X86_ANY) || defined(VM_ARM_ANY)
#elif defined(_MSC_VER)

static INLINE void *
uint16set(void *dst, uint16 val, size_t count)
{
#ifdef VM_X86_64
   __stosw((uint16*)dst, val, count);
#elif defined(VM_ARM_32)
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint16 *)dst)[i] = val;
   }
#else
   __asm { pushf;
           mov ax, val;
           mov ecx, count;
           mov edi, dst;
           cld;
           rep stosw;
           popf;
   }
#endif
   return dst;
}

static INLINE void *
uint32set(void *dst, uint32 val, size_t count)
{
#ifdef VM_X86_64
   __stosd((unsigned long*)dst, (unsigned long)val, count);
#elif defined(VM_ARM_32)
   size_t i;
   for (i = 0; i < count; i++) {
      ((uint32 *)dst)[i] = val;
   }
#else
   __asm { pushf;
           mov eax, val;
           mov ecx, count;
           mov edi, dst;
           cld;
           rep stosd;
           popf;
   }
#endif
   return dst;
}

#else
#error "No compiler defined for uint*set"
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * Bswap16 --
 *
 *      Swap the 2 bytes of "v" as follows: 32 -> 23.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint16
Bswap16(uint16 v)
{
#if defined(VM_ARM_64)
   __asm__("rev16 %w0, %w0" : "+r"(v));
   return v;
#else
   return ((v >> 8) & 0x00ff) | ((v << 8) & 0xff00);
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * Bswap32 --
 *
 *      Swap the 4 bytes of "v" as follows: 3210 -> 0123.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint32
Bswap32(uint32 v) // IN
{
#if defined(__GNUC__) && defined(VM_X86_ANY)
   /* Checked against the Intel manual and GCC. --hpreg */
   __asm__(
      "bswap %0"
      : "=r" (v)
      : "0" (v)
   );
   return v;
#elif defined(VM_ARM_32) && !defined(__ANDROID__) && !defined(_MSC_VER)
    __asm__("rev %0, %0" : "+r"(v));
    return v;
#elif defined(VM_ARM_64)
   __asm__("rev32 %x0, %x0" : "+r"(v));
   return v;
#else
   return    (v >> 24)
          | ((v >>  8) & 0xFF00)
          | ((v & 0xFF00) <<  8)
          |  (v << 24)          ;
#endif
}
#define Bswap Bswap32

/*
 *-----------------------------------------------------------------------------
 *
 * Bswap64 --
 *
 *      Swap the 8 bytes of "v" as follows: 76543210 -> 01234567.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE uint64
Bswap64(uint64 v) // IN
{
#if defined(VM_ARM_64)
   __asm__("rev %0, %0" : "+r"(v));
   return v;
#else
   return ((uint64)Bswap((uint32)v) << 32) | Bswap((uint32)(v >> 32));
#endif
}

/*
 * PAUSE is a P4 instruction that improves spinlock power+performance;
 * on non-P4 IA32 systems, the encoding is interpreted as a REPZ-NOP.
 * Use volatile to avoid NOP removal.
 */
static INLINE void
PAUSE(void)
#if defined(__GNUC__) || defined(VM_ARM_32)
{
#ifdef VM_ARM_ANY
   /*
    * ARM has no instruction to execute "spin-wait loop", just leave it
    * empty.
    */
#else
   __asm__ __volatile__( "pause" :);
#endif
}
#elif defined(_MSC_VER)
#ifdef VM_X86_64
{
   _mm_pause();
}
#else /* VM_X86_64 */
#pragma warning( disable : 4035)
{
   __asm _emit 0xf3 __asm _emit 0x90
}
#pragma warning (default: 4035)
#endif /* VM_X86_64 */
#else  /* __GNUC__  */
#error No compiler defined for PAUSE
#endif

/*
 * Checked against the Intel manual and GCC --hpreg
 *
 * volatile because the tsc always changes without the compiler knowing it.
 */
static INLINE uint64
RDTSC(void)
#ifdef __GNUC__
{
#ifdef VM_X86_64
   uint64 tscLow;
   uint64 tscHigh;

__asm__ __volatile__(
      "rdtsc"
      : "=a" (tscLow), "=d" (tscHigh)
   );

return tscHigh << 32 | tscLow;
#elif defined(VM_X86_32)
   uint64 tim;

__asm__ __volatile__(
      "rdtsc"
      : "=A" (tim)
   );

return tim;
#elif defined(VM_ARM_64)
#if (defined(VMKERNEL) || defined(VMM)) && !defined(VMK_ARM_EL1)
   return MRS(CNTPCT_EL0);
#else
   return MRS(CNTVCT_EL0);
#endif
#else
   /*
    * For platform without cheap timer, just return 0.
    */
   return 0;
#endif
}
#elif defined(_MSC_VER)
#ifdef VM_X86_64
{
   return __rdtsc();
}
#elif defined(VM_ARM_32)
{
   /*
    * We need to do more inverstagetion here to find
    * a microsoft equivalent of that code
    */
   NOT_IMPLEMENTED();
   return 0;
}
#else
#pragma warning( disable : 4035)
{
   __asm _emit 0x0f __asm _emit 0x31
}
#pragma warning (default: 4035)
#endif /* VM_X86_64 */
#else  /* __GNUC__  */
#error No compiler defined for RDTSC
#endif /* __GNUC__  */

/*
 *-----------------------------------------------------------------------------
 *
 * DEBUGBREAK --
 *
 *    Does an int3 for MSVC / GCC, bkpt/brk for ARM. This is a macro to make
 *    sure int3 is always inlined.
 *
 *-----------------------------------------------------------------------------
 */

#ifdef VM_ARM_32
#define DEBUGBREAK() __asm__("bkpt")
#elif defined(VM_ARM_64)
#define DEBUGBREAK() __asm__("brk #0")
#elif defined(_MSC_VER)
#define DEBUGBREAK() __debugbreak()
#else
#define DEBUGBREAK() __asm__("int $3")
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * {Clear,Set,Test}Bit{32,64} --
 *
 *    Sets or clears a specified single bit in the provided variable.
 *
 *    The index input value specifies which bit to modify and is 0-based.
 *    Index is truncated by hardware to a 5-bit or 6-bit offset for the
 *    32 and 64-bit flavors, respectively, but input values are not validated
 *    with asserts to avoid include dependencies.
 *
 *    64-bit flavors are not handcrafted for 32-bit builds because they may
 *    defeat compiler optimizations.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE void
SetBit32(uint32 *var, uint32 index)
{
#if defined(__GNUC__) && defined(VM_X86_ANY)
   __asm__ (
      "bts %1, %0"
      : "+mr" (*var)
      : "rI" (index)
      : "cc"
   );
#elif defined(_MSC_VER)
   _bittestandset((long *)var, index);
#else
   *var |= (1 << index);
#endif
}

static INLINE void
ClearBit32(uint32 *var, uint32 index)
{
#if defined(__GNUC__) && defined(VM_X86_ANY)
   __asm__ (
      "btr %1, %0"
      : "+mr" (*var)
      : "rI" (index)
      : "cc"
   );
#elif defined(_MSC_VER)
   _bittestandreset((long *)var, index);
#else
   *var &= ~(1 << index);
#endif
}

static INLINE void
SetBit64(uint64 *var, uint64 index)
{
#if defined(VM_64BIT) && !defined(VM_ARM_64)
#ifdef __GNUC__
   __asm__ (
      "bts %1, %0"
      : "+mr" (*var)
      : "rJ" (index)
      : "cc"
   );
#elif defined(_MSC_VER)
   _bittestandset64((__int64 *)var, index);
#endif
#else
   *var |= ((uint64)1 << index);
#endif
}

static INLINE void
ClearBit64(uint64 *var, uint64 index)
{
#if defined(VM_64BIT) && !defined(VM_ARM_64)
#ifdef __GNUC__
   __asm__ (
      "btrq %1, %0"
      : "+mr" (*var)
      : "rJ" (index)
      : "cc"
   );
#elif defined(_MSC_VER)
   _bittestandreset64((__int64 *)var, index);
#endif
#else
   *var &= ~((uint64)1 << index);
#endif
}

static INLINE Bool
TestBit32(const uint32 *var, uint32 index)
{
#if defined(__GNUC__) && defined(VM_X86_ANY)
   Bool bit;
   __asm__ (
      "bt %[index], %[var] \n"
      "setc %[bit]"
      : [bit] "=qQm" (bit)
      : [index] "rI" (index), [var] "r" (*var)
      : "cc"
   );
   return bit;
#else
   return (*var & (1 << index)) != 0;
#endif
}

static INLINE Bool
TestBit64(const uint64 *var, uint64 index)
{
#if defined __GNUC__ && defined VM_X86_64
   Bool bit;
   __asm__ (
      "bt %[index], %[var] \n"
      "setc %[bit]"
      : [bit] "=qQm" (bit)
      : [index] "rJ" (index), [var] "r" (*var)
      : "cc"
   );
   return bit;
#else
   return (*var & (CONST64U(1) << index)) != 0;
#endif
}

/*
 *-----------------------------------------------------------------------------
 *
 * {Clear,Set,Complement,Test}BitVector --
 *
 *    Sets, clears, complements, or tests a specified single bit in the
 *    provided array.  The index input value specifies which bit to modify
 *    and is 0-based.  Bit number can be +-2Gb (+-128MB) relative from 'var'
 *    variable.
 *
 *    All functions return value of the bit before modification was performed.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE Bool
SetBitVector(void *var, int32 index)
{
#if defined(__GNUC__) && defined(VM_X86_ANY)
   Bool bit;
   __asm__ (
      "bts %2, %1;"
      "setc %0"
      : "=qQm" (bit), "+m" (*(uint32 *)var)
      : "rI" (index)
      : "memory", "cc"
   );
   return bit;
#elif defined(_MSC_VER)
   return _bittestandset((long *)var, index) != 0;
#else
   Bool retVal = (((uint8 *)var)[index / 8] & (1 << (index % 8))) != 0;
   ((uint8 *)var)[index / 8] |= 1 << (index % 8);
   return retVal;
#endif
}

static INLINE Bool
ClearBitVector(void *var, int32 index)
{
#if defined(__GNUC__) && defined(VM_X86_ANY)
   Bool bit;
   __asm__ (
      "btr %2, %1;"
      "setc %0"
      : "=qQm" (bit), "+m" (*(uint32 *)var)
      : "rI" (index)
      : "memory", "cc"
   );
   return bit;
#elif defined(_MSC_VER)
   return _bittestandreset((long *)var, index) != 0;
#else
   Bool retVal = (((uint8 *)var)[index / 8] & (1 << (index % 8))) != 0;
   ((uint8 *)var)[index / 8] &= ~(1 << (index % 8));
   return retVal;
#endif
}

static INLINE Bool
ComplementBitVector(void *var, int32 index)
{
#if defined(__GNUC__) && defined(VM_X86_ANY)
   Bool bit;
   __asm__ (
      "btc %2, %1;"
      "setc %0"
      : "=qQm" (bit), "+m" (*(uint32 *)var)
      : "rI" (index)
      : "memory", "cc"
   );
   return bit;
#elif defined(_MSC_VER)
   return _bittestandcomplement((long *)var, index) != 0;
#else
   Bool retVal = (((uint8 *)var)[index / 8] & (1 << (index % 8))) != 0;
   ((uint8 *)var)[index / 8] ^= ~(1 << (index % 8));
   return retVal;
#endif
}

static INLINE Bool
TestBitVector(const void *var, int32 index)
{
#if defined(__GNUC__) && defined(VM_X86_ANY)
   Bool bit;
   __asm__ (
      "bt %2, %1;"
      "setc %0"
      : "=qQm" (bit)
      : "m" (*(const uint32 *)var), "rI" (index)
      : "cc"
   );
   return bit;
#elif defined _MSC_VER
   return _bittest((long *)var, index) != 0;
#else
   return (((const uint8 *)var)[index / 8] & (1 << (index % 8))) != 0;
#endif
}

/*
 *-----------------------------------------------------------------------------
 * RoundUpPow2_{64,32} --
 *
 *   Rounds a value up to the next higher power of 2.  Returns the original
 *   value if it is a power of 2.  The next power of 2 for inputs {0, 1} is 1.
 *   The result is undefined for inputs above {2^63, 2^31} (but equal to 1
 *   in this implementation).
 *-----------------------------------------------------------------------------
 */

static INLINE uint64
RoundUpPow2C64(uint64 value)
{
   if (value <= 1 || value > (CONST64U(1) << 63)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
      return (CONST64U(2) << mssb64_0(value - 1));
   }
}

#if defined(__GNUC__) && defined(VM_X86_64)
static INLINE uint64
RoundUpPow2Asm64(uint64 value)
{
   uint64 out = 2;
   __asm__("lea -1(%[in]), %%rcx;"      // rcx = value - 1.  Preserve original.
           "bsr %%rcx, %%rcx;"          // rcx = log2(value - 1) if value != 1
                                        // if value == 0, then rcx = 63
                                        // if value == 1 then zf = 1, else zf = 0.
           "rol %%cl, %[out];"          // out = 2 << rcx (if rcx != -1)
                                        //     = 2^(log2(value - 1) + 1)
                                        // if rcx == -1 (value == 0), out = 1
                                        // zf is always unmodified.
           "cmovz %[in], %[out]"        // if value == 1 (zf == 1), write 1 to out.
       : [out]"+r"(out) : [in]"r"(value) : "%rcx", "cc");
   return out;
}
#endif

static INLINE uint64
RoundUpPow2_64(uint64 value)
{
#if defined(__GNUC__) && defined(VM_X86_64)
   if (__builtin_constant_p(value)) {
      return RoundUpPow2C64(value);
   } else {
      return RoundUpPow2Asm64(value);
   }
#else
   return RoundUpPow2C64(value);
#endif
}

static INLINE uint32
RoundUpPow2C32(uint32 value)
{
   if (value <= 1 || value > (1U << 31)) {
      return 1; // Match the assembly's undefined value for large inputs.
   } else {
      return (2 << mssb32_0(value - 1));
   }
}

#ifdef __GNUC__
static INLINE uint32
RoundUpPow2Asm32(uint32 value)
{
#ifdef VM_ARM_32
   uint32 out = 1;
   // Note: None Thumb only!
   //       The value of the argument "value"
   //       will be affected!
   __asm__("sub %[in], %[in], #1;"         // r1 = value - 1 . if value == 0 then r1 = 0xFFFFFFFF
           "clz %[in], %[in];"             // r1 = log2(value - 1) if value != 1
                                           // if value == 0 then r1 = 0
                                           // if value == 1 then r1 = 32
           "mov %[out], %[out], ror %[in]" // out = 2^(32 - r1)
                                           // if out == 2^32 then out = 1 as it is right rotate
       : [in]"+r"(value),[out]"+r"(out));
   return out;
#elif defined(VM_ARM_64)
   return RoundUpPow2C32(value);
#else
   uint32 out = 2;

__asm__("lea -1(%[in]), %%ecx;"      // ecx = value - 1.  Preserve original.
           "bsr %%ecx, %%ecx;"          // ecx = log2(value - 1) if value != 1
                                        // if value == 0, then ecx = 31
                                        // if value == 1 then zf = 1, else zf = 0.
           "rol %%cl, %[out];"          // out = 2 << ecx (if ecx != -1)
                                        //     = 2^(log2(value - 1) + 1).
                                        // if ecx == -1 (value == 0), out = 1
                                        // zf is always unmodified
           "cmovz %[in], %[out]"        // if value == 1 (zf == 1), write 1 to out.
       : [out]"+r"(out) : [in]"r"(value) : "%ecx", "cc");
   return out;
#endif
}
#endif // __GNUC__

static INLINE uint32
RoundUpPow2_32(uint32 value)
{
#ifdef __GNUC__
   if (__builtin_constant_p(value)) {
      return RoundUpPow2C32(value);
   } else {
      return RoundUpPow2Asm32(value);
   }
#else
   return RoundUpPow2C32(value);
#endif
}

#if defined __cplusplus
} // extern "C"
#endif

#endif // _VM_BASIC_ASM_H_

���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vmciKernelAPI3.h����������������������������������������������������������������0000444�0000000�0000000�00000003150�13432725350�016273� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmciKernelAPI3.h --
 *
 *    Kernel API (v3) exported from the VMCI host and guest drivers.
 */

#ifndef __VMCI_KERNELAPI_3_H__
#define __VMCI_KERNELAPI_3_H__

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#include "includeCheck.h"

#include "vmciKernelAPI2.h"

#if defined __cplusplus
extern "C" {
#endif

/* Define version 3. */

#undef  VMCI_KERNEL_API_VERSION
#define VMCI_KERNEL_API_VERSION_3 3
#define VMCI_KERNEL_API_VERSION   VMCI_KERNEL_API_VERSION_3

/* VMCI Detach Cause API (only available in vmkernel). */

#define VMCI_DETACH_REGULAR  0
#define VMCI_DETACH_VMOTION  1

int vmci_qpair_get_detach_cause(VMCIQPair *qpair, uint8 *cause);

#if defined __cplusplus
} // extern "C"
#endif

#endif /* !__VMCI_KERNELAPI_3_H__ */
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/guest_msg_def.h�����������������������������������������������������������������0000444�0000000�0000000�00000005647�13432725350�016407� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998-2016 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * guest_msg_def.h --
 *
 *    Second layer of the internal communication channel between guest
 *    applications and vmware
 *
 */

#ifndef _GUEST_MSG_DEF_H_
#define _GUEST_MSG_DEF_H_

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_USERLEVEL

#include "includeCheck.h"

/* Basic request types */
typedef enum {
   MESSAGE_TYPE_OPEN,
   MESSAGE_TYPE_SENDSIZE,
   MESSAGE_TYPE_SENDPAYLOAD,
   MESSAGE_TYPE_RECVSIZE,
   MESSAGE_TYPE_RECVPAYLOAD,
   MESSAGE_TYPE_RECVSTATUS,
   MESSAGE_TYPE_CLOSE,
} MessageType;

/* Reply statuses */
/*  The basic request succeeded */
#define MESSAGE_STATUS_SUCCESS  0x0001
/*  vmware has a message available for its party */
#define MESSAGE_STATUS_DORECV   0x0002
/*  The channel has been closed */
#define MESSAGE_STATUS_CLOSED   0x0004
/*  vmware removed the message before the party fetched it */
#define MESSAGE_STATUS_UNSENT   0x0008
/*  A checkpoint occurred */
#define MESSAGE_STATUS_CPT      0x0010
/*  An underlying device is powering off */
#define MESSAGE_STATUS_POWEROFF 0x0020
/*  vmware has detected a timeout on the channel */
#define MESSAGE_STATUS_TIMEOUT  0x0040
/*  vmware supports high-bandwidth for sending and receiving the payload */
#define MESSAGE_STATUS_HB       0x0080

/*
 * This mask defines the status bits that the guest is allowed to set;
 * we use this to mask out all other bits when receiving the status
 * from the guest. Otherwise, the guest can manipulate VMX state by
 * setting status bits that are only supposed to be changed by the
 * VMX. See bug 45385.
 */
#define MESSAGE_STATUS_GUEST_MASK    MESSAGE_STATUS_SUCCESS

/*
 * Max number of channels.
 * Unfortunately this has to be public because the monitor part
 * of the backdoor needs it for its trivial-case optimization. [greg]
 */
#define GUESTMSG_MAX_CHANNEL 8

/* Flags to open a channel. --hpreg */
#define GUESTMSG_FLAG_COOKIE 0x80000000
#define GUESTMSG_FLAG_ALL GUESTMSG_FLAG_COOKIE

/*
 * Maximum size of incoming message. This is to prevent denial of host service
 * attacks from guest applications.
 */
#define GUESTMSG_MAX_IN_SIZE (64 * 1024)

#endif /* _GUEST_MSG_DEF_H_ */
�����������������������������������������������������������������������������������������vmxnet3-only/shared/dbllnklst.h���������������������������������������������������������������������0000444�0000000�0000000�00000015155�13432725346�015565� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * dbllnklst.h --
 *
 *    Double linked lists
 */

#ifndef _DBLLNKLST_H_
#define _DBLLNKLST_H_

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_USERLEVEL
#include "includeCheck.h"

#include "vm_basic_types.h"

#if defined(__cplusplus)
extern "C" {
#endif

#define DblLnkLst_OffsetOf(type, field) ((intptr_t)&((type *)0)->field)

#define DblLnkLst_Container(addr, type, field) \
   ((type *)((char *)(addr) - DblLnkLst_OffsetOf(type, field)))

#define DblLnkLst_ForEach(curr, head)                   \
      for (curr = (head)->next; curr != (head); curr = (curr)->next)

/* Safe from list element removal within loop body. */
#define DblLnkLst_ForEachSafe(curr, nextElem, head)             \
      for (curr = (head)->next, nextElem = (curr)->next;        \
           curr != (head);                                      \
           curr = nextElem, nextElem = (curr)->next)

typedef struct DblLnkLst_Links {
   struct DblLnkLst_Links *prev;
   struct DblLnkLst_Links *next;
} DblLnkLst_Links;

/*
 * Functions
 *
 * DblLnkLst_LinkFirst, DblLnkLst_LinkLast, and DblLnkLst_Swap are specific
 * to anchored lists.  The rest are for both circular and anchored lists.
 */

/*
 *----------------------------------------------------------------------
 *
 * DblLnkLst_Init --
 *
 *    Initialize a member of a doubly linked list
 *
 * Result
 *    None
 *
 * Side effects:
 *    None
 *
 *----------------------------------------------------------------------
 */

static INLINE void
DblLnkLst_Init(DblLnkLst_Links *l) // OUT
{
   l->prev = l->next = l;
}

/*
 *----------------------------------------------------------------------
 *
 * DblLnkLst_Link --
 *
 *    Merge two doubly linked lists into one
 *
 *    The operation is commutative
 *    The operation is inversible (its inverse is DblLnkLst_Unlink)
 *
 * Result
 *    None
 *
 * Side effects:
 *    None
 *
 *----------------------------------------------------------------------
 */

static INLINE void
DblLnkLst_Link(DblLnkLst_Links *l1, // IN/OUT
               DblLnkLst_Links *l2) // IN/OUT
{
   DblLnkLst_Links *tmp;

(tmp      = l1->prev)->next = l2;
   (l1->prev = l2->prev)->next = l1;
    l2->prev = tmp                 ;
}

/*
 *----------------------------------------------------------------------
 *
 * DblLnkLst_Unlink --
 *
 *    Split one doubly linked list into two
 *
 *    No check is performed: the caller must ensure that both members
 *    belong to the same doubly linked list
 *
 *    The operation is commutative
 *    The operation is inversible (its inverse is DblLnkLst_Link)
 *
 * Result
 *    None
 *
 * Side effects:
 *    None
 *
 *----------------------------------------------------------------------
 */

static INLINE void
DblLnkLst_Unlink(DblLnkLst_Links *l1, // IN/OUT
                 DblLnkLst_Links *l2) // IN/OUT
{
   DblLnkLst_Links *tmp;

tmp       = l1->prev            ;
   (l1->prev = l2->prev)->next = l1;
   (l2->prev = tmp     )->next = l2;
}

/*
 *----------------------------------------------------------------------
 *
 * DblLnkLst_Unlink1 --
 *
 *    Unlink an element from its list.
 *
 * Result
 *    None
 *
 * Side effects:
 *    None
 *
 *----------------------------------------------------------------------
 */

static INLINE void
DblLnkLst_Unlink1(DblLnkLst_Links *l) // IN/OUT
{
   DblLnkLst_Unlink(l, l->next);
}

/*
 *----------------------------------------------------------------------------
 *
 * DblLnkLst_IsLinked --
 *
 *    Determines whether an element is linked with any other elements.
 *
 * Results:
 *    TRUE if link is linked, FALSE otherwise.
 *
 * Side effects:
 *    None.
 *
 *----------------------------------------------------------------------------
 */

static INLINE Bool
DblLnkLst_IsLinked(DblLnkLst_Links const *l) // IN
{
   /*
    * A DblLnkLst_Links is either linked to itself (not linked) or linked to
    * other elements in a list (linked).
    */
   return l->prev != l;
}

/*
 *----------------------------------------------------------------------
 *
 * DblLnkLst_LinkFirst --
 *
 *    Insert 'l' at the beginning of the list anchored at 'head'
 *
 * Result
 *    None
 *
 * Side effects:
 *    None
 *
 *----------------------------------------------------------------------
 */

static INLINE void
DblLnkLst_LinkFirst(DblLnkLst_Links *head, // IN/OUT
                    DblLnkLst_Links *l)    // IN/OUT
{
   DblLnkLst_Link(head->next, l);
}

/*
 *----------------------------------------------------------------------
 *
 * DblLnkLst_LinkLast --
 *
 *    Insert 'l' at the end of the list anchored at 'head'
 *
 * Result
 *    None
 *
 * Side effects:
 *    None
 *
 *----------------------------------------------------------------------
 */

static INLINE void
DblLnkLst_LinkLast(DblLnkLst_Links *head, // IN/OUT
                   DblLnkLst_Links *l)    // IN/OUT
{
   DblLnkLst_Link(head, l);
}

/*
 *----------------------------------------------------------------------
 *
 * DblLnkLst_Swap --
 *
 *    Swap all entries between the list anchored at 'head1' and the list
 *    anchored at 'head2'.
 *
 *    The operation is commutative
 *    The operation is inversible (its inverse is itself)
 *
 * Result
 *    None
 *
 * Side effects:
 *    None
 *
 *----------------------------------------------------------------------
 */

static INLINE void
DblLnkLst_Swap(DblLnkLst_Links *head1,  // IN/OUT
               DblLnkLst_Links *head2)  // IN/OUT
{
   DblLnkLst_Links const tmp = *head1;

if (DblLnkLst_IsLinked(head2)) {
      (head1->prev = head2->prev)->next = head1;
      (head1->next = head2->next)->prev = head1;
   } else {
      DblLnkLst_Init(head1);
   }

if (tmp.prev != head1) {
      (head2->prev = tmp.prev)->next = head2;
      (head2->next = tmp.next)->prev = head2;
   } else {
      DblLnkLst_Init(head2);
   }
}

#if defined(__cplusplus)
}  // extern "C"
#endif

#endif /* _DBLLNKLST_H_ */
�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/compat_scsi.h�������������������������������������������������������������������0000444�0000000�0000000�00000003024�13432725347�016071� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_SCSI_H__
#   define __COMPAT_SCSI_H__

/* The scsi_bufflen() API appeared somewhere in time --hpreg */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23)
#   define scsi_bufflen(cmd) ((cmd)->request_bufflen)
#   define scsi_sg_count(cmd) ((cmd)->use_sg)
#   define scsi_sglist(cmd) ((struct scatterlist *)(cmd)->request_buffer)
#   define scsi_set_resid(cmd, _resid) ((cmd)->resid = _resid)
#endif

/*
 * Using scsi_sglist to access the request buffer looks strange
 * so instead we define this macro.  What happened is later kernel
 * put all SCSI data in sglists, since it simplifies passing buffers
 */
#define scsi_request_buffer(cmd) scsi_sglist(cmd)

#endif /* __COMPAT_SCSI_H__ */
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/vm_basic_math.h�����������������������������������������������������������������0000444�0000000�0000000�00000010721�13432725350�016355� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2008-2017 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vm_basic_math.h --
 *
 *	Standard mathematical macros for VMware source code.
 */

#ifndef _VM_BASIC_MATH_H_
#define _VM_BASIC_MATH_H_

#define INCLUDE_ALLOW_USERLEVEL

#if defined __cplusplus
extern "C" {
#endif

static INLINE uint32
RatioOf(uint32 numer1, uint32 numer2, uint32 denom)
{
   uint64 numer = (uint64)numer1 * numer2;
   /* Calculate "(numer1 * numer2) / denom" avoiding round-off errors. */
#if defined(VMM) || !(defined(__i386__) || defined(__x86_64__))
   return numer / denom;
#else
   uint32 ratio;
   uint32 unused;
   Div643232(numer, denom, &ratio, &unused);
   return ratio;
#endif
}

static INLINE uint32
ExponentialAvg(uint32 avg, uint32 value, uint32 gainNumer, uint32 gainDenom)
{
   uint32 term1 = gainNumer * avg;
   uint32 term2 = (gainDenom - gainNumer) * value;
   return (term1 + term2) / gainDenom;
}

/*
 *-----------------------------------------------------------------------------
 *
 * IsZeroOrPowerOfTwo --
 * IsZeroOrPowerOfTwo64 --
 *
 * Results:
 *      TRUE iff the value is 0 or a power of two.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */

static INLINE Bool
IsZeroOrPowerOfTwo64(uint64 x)
{
   return !(x & (x - 1));
}

static INLINE Bool
IsZeroOrPowerOfTwo(uint32 x)     // IN
{
   return !(x & (x - 1));
}

static INLINE uint32
GetPowerOfTwo(uint32 x)
{
   /* Returns next-greatest power-of-two. */
   uint32 power2 = 1;
   while (x > power2) {
      power2 = power2 << 1;
   }
   return power2;
}

#if !defined(_WIN32) && !defined(_WIN64)
/*
 *-----------------------------------------------------------------------------
 *
 * RotateLeft32 --
 *
 * Results:
 *      Value rotated to the left by 'shift' bits.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */
static INLINE uint32
RotateLeft32(uint32 value, uint8 shift)
{
   return ((value << shift) | (value >> (32 - shift)));
}

/*
 *-----------------------------------------------------------------------------
 *
 * RotateRight32 --
 *
 * Results:
 *      Value rotated to the right by 'shift' bits.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */
static INLINE uint32
RotateRight32(uint32 value, uint8 shift)
{
   return ((value >> shift) | (value << (32 - shift)));
}

/*
 *-----------------------------------------------------------------------------
 *
 * RotateLeft64 --
 *
 * Results:
 *      Value rotated to the left by 'shift' bits.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */
static INLINE uint64
RotateLeft64(uint64 value, uint8 shift)
{
   return ((value << shift) | (value >> (64 - shift)));
}

/*
 *-----------------------------------------------------------------------------
 *
 * RotateRight64 --
 *
 * Results:
 *      Value rotated to the right by 'shift' bits.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */
static INLINE uint64
RotateRight64(uint64 value, uint8 shift)
{
   return ((value >> shift) | (value << (64 - shift)));
}
#endif // if !defined(_WIN32) && !defined(_WIN64)

#if defined __cplusplus
} // extern "C"
#endif

#endif // ifndef _VM_BASIC_MATH_H_
�����������������������������������������������vmxnet3-only/shared/compat_slab.h�������������������������������������������������������������������0000444�0000000�0000000�00000006653�13432725347�016064� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2005 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_SLAB_H__
#   define __COMPAT_SLAB_H__

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 2, 0)
#   include <linux/slab.h>
#else
#   include <linux/malloc.h>
#endif

/*
 * Before 2.6.20, kmem_cache_t was the accepted way to refer to a kmem_cache
 * structure.  Prior to 2.6.15, this structure was called kmem_cache_s, and
 * afterwards it was renamed to kmem_cache.  Here we keep things simple and use
 * the accepted typedef until it became deprecated, at which point we switch
 * over to the kmem_cache name.
 */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20)
#   define compat_kmem_cache struct kmem_cache
#else
#   define compat_kmem_cache kmem_cache_t
#endif

/*
 * Up to 2.6.22 kmem_cache_create has 6 arguments - name, size, alignment, flags,
 * constructor, and destructor.  Then for some time kernel was asserting that
 * destructor is NULL, and since 2.6.23-pre1 kmem_cache_create takes only 5
 * arguments - destructor is gone.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 22) || defined(VMW_KMEMCR_HAS_DTOR)
#define compat_kmem_cache_create(name, size, align, flags, ctor) \
		kmem_cache_create(name, size, align, flags, ctor, NULL)
#else
#define compat_kmem_cache_create(name, size, align, flags, ctor) \
		kmem_cache_create(name, size, align, flags, ctor)
#endif

/*
 * Up to 2.6.23 kmem_cache constructor has three arguments - pointer to block to
 * prepare (aka "this"), from which cache it came, and some unused flags.  After
 * 2.6.23 flags were removed, and order of "this" and cache parameters was swapped...
 * Since 2.6.27-rc2 everything is different again, and ctor has only one argument.
 *
 * HAS_3_ARGS has precedence over HAS_2_ARGS if both are defined.
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 23) && !defined(VMW_KMEMCR_CTOR_HAS_3_ARGS)
#  define VMW_KMEMCR_CTOR_HAS_3_ARGS
#endif
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 26) && !defined(VMW_KMEMCR_CTOR_HAS_2_ARGS)
#  define VMW_KMEMCR_CTOR_HAS_2_ARGS
#endif

#if defined(VMW_KMEMCR_CTOR_HAS_3_ARGS)
typedef void compat_kmem_cache_ctor(void *, compat_kmem_cache *, unsigned long);
#define COMPAT_KMEM_CACHE_CTOR_ARGS(arg) void *arg, \
                                         compat_kmem_cache *cache, \
                                         unsigned long flags
#elif defined(VMW_KMEMCR_CTOR_HAS_2_ARGS)
typedef void compat_kmem_cache_ctor(compat_kmem_cache *, void *);
#define COMPAT_KMEM_CACHE_CTOR_ARGS(arg) compat_kmem_cache *cache, \
                                         void *arg
#else
typedef void compat_kmem_cache_ctor(void *);
#define COMPAT_KMEM_CACHE_CTOR_ARGS(arg) void *arg
#endif

#endif /* __COMPAT_SLAB_H__ */
�������������������������������������������������������������������������������������vmxnet3-only/shared/compat_mm.h���������������������������������������������������������������������0000444�0000000�0000000�00000003002�13432725347�015535� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2002 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_MM_H__
#   define __COMPAT_MM_H__

#include <linux/mm.h>

/* 2.2.x uses 0 instead of some define */
#ifndef NOPAGE_SIGBUS
#define NOPAGE_SIGBUS (0)
#endif

/* 2.2.x does not have HIGHMEM support */
#ifndef GFP_HIGHUSER
#define GFP_HIGHUSER (GFP_USER)
#endif

/*
 * In 2.4.14, the logic behind the UnlockPage macro was moved to the
 * unlock_page() function. Later (in 2.5.12), the UnlockPage macro was removed
 * altogether, and nowadays everyone uses unlock_page().
 */
#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 4, 14)
#define compat_unlock_page(page) UnlockPage(page)
#else
#define compat_unlock_page(page) unlock_page(page)
#endif

#endif /* __COMPAT_MM_H__ */
������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/shared/driver-config.h�����������������������������������������������������������������0000444�0000000�0000000�00000004314�13432725347�016326� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1998 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * Sets the proper defines from the Linux header files
 *
 * This file must be included before the inclusion of any kernel header file,
 * with the exception of linux/autoconf.h and linux/version.h --hpreg
 */

#ifndef __VMX_CONFIG_H__
#define __VMX_CONFIG_H__

#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMKDRIVERS
#include "includeCheck.h"

#include "compat_version.h"
#include "compat_autoconf.h"

/*
 * We rely on Kernel Module support.  Check here.
 */
#ifndef CONFIG_MODULES
#   error "No Module support in this kernel.  Please configure with CONFIG_MODULES"
#endif

/*
 * 2.2 kernels still use __SMP__ (derived from CONFIG_SMP
 * in the main Makefile), so we do it here.
 */

#ifdef CONFIG_SMP
#   define __SMP__ 1
#endif

#if defined(CONFIG_MODVERSIONS) && defined(KERNEL_2_1)
#   if LINUX_VERSION_CODE < KERNEL_VERSION(2,5,60)
/*
 * MODVERSIONS might be already defined when using kernel's Makefiles.
 */
#      ifndef MODVERSIONS
#         define MODVERSIONS
#      endif
#      include <linux/modversions.h>
#   endif
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 24)
/*
 * Force the uintptr_t definition to come from linux/types.h instead of vm_basic_types.h.
 */
#   include <linux/types.h>
#   define _STDINT_H 1
#endif

#ifndef __KERNEL__
#   define __KERNEL__
#endif

/*
 * x86cpuid_asm.h
 *
 *      CPUID-related assembly functions.
 */

#ifndef _X86CPUID_ASM_H_
#define _X86CPUID_ASM_H_

#define INCLUDE_ALLOW_USERLEVEL

#include "vm_basic_asm.h"
#include "x86cpuid.h"

/*
 * x86-64 windows doesn't support inline asm so we have to use these
 * intrinsic functions defined in the compiler.  Not all of these are well
 * documented.  There is an array in the compiler dll (c1.dll) which has
 * an array of the names of all the intrinsics minus the leading
 * underscore.  Searching around in the ntddk.h file can also be helpful.
 *
 * The declarations for the intrinsic functions were taken from the DDK.
 * Our declarations must match the ddk's otherwise the 64-bit c++ compiler
 * will complain about second linkage of the intrinsic functions.
 * We define the intrinsic using the basic types corresponding to the
 * Windows typedefs. This avoids having to include windows header files
 * to get to the windows types.
 */
#ifdef _MSC_VER
#ifdef __cplusplus
extern "C" {
#endif
#ifdef VM_X86_64
/*
 * intrinsic functions only supported by x86-64 windows as of 2k3sp1
 */
void __cpuid(int regs[4], int eax);
#pragma intrinsic(__cpuid)

/*
 * __cpuidex has been supported since VS2008
 */
#if _MSC_VER >= 1500
void __cpuidex(int regs[4], int eax, int ecx);
#pragma intrinsic(__cpuidex)
#endif /* _MSC_VER >= 1500 */
#endif /* VM_X86_64 */

#ifdef __cplusplus
}
#endif
#endif /* _MSC_VER */

#ifdef __GNUC__ // {

/*
 * Checked against the Intel manual and GCC --hpreg
 *
 * Need __volatile__ and "memory" since CPUID has a synchronizing effect.
 * The CPUID may also change at runtime (APIC flag, etc).
 *
 */

/*
 * %ebx is reserved on i386 PIC.  Apple's gcc-5493 (gcc 4.0) compiling
 * for x86_64 incorrectly errors out saying %ebx is reserved.  This is
 * Apple bug 7304232.
 */
#if vm_x86_64 ? (defined __APPLE_CC__ && __APPLE_CC__ == 5493) : defined __PIC__
#if vm_x86_64
/*
 * Note that this generates movq %rbx,%rbx; cpuid; xchgq %rbx,%rbx ...
 * Unfortunately Apple's assembler does not have .ifnes, and I cannot
 * figure out how to do that with .if.   If we ever enable this code
 * on other 64bit systems, both movq & xchgq should be surrounded by
 * .ifnes \"%%rbx\", \"%q1\" & .endif
 */
#define VM_CPUID_BLOCK  "movq %%rbx, %q1\n\t" \
                        "cpuid\n\t"           \
                        "xchgq %%rbx, %q1\n\t"
#define VM_EBX_OUT(reg) "=&r"(reg)
#else
#define VM_CPUID_BLOCK  "movl %%ebx, %1\n\t" \
                        "cpuid\n\t"          \
                        "xchgl %%ebx, %1\n\t"
#define VM_EBX_OUT(reg) "=&rm"(reg)
#endif
#else
#define VM_CPUID_BLOCK  "cpuid"
#define VM_EBX_OUT(reg) "=b"(reg)
#endif

static INLINE void
__GET_CPUID(int eax,         // IN
            CPUIDRegs *regs) // OUT
{
   __asm__ __volatile__(
      VM_CPUID_BLOCK
      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
      : "a" (eax)
      : "memory"
   );
}

static INLINE void
__GET_CPUID2(int eax,         // IN
             int ecx,         // IN
             CPUIDRegs *regs) // OUT
{
   __asm__ __volatile__(
      VM_CPUID_BLOCK
      : "=a" (regs->eax), VM_EBX_OUT(regs->ebx), "=c" (regs->ecx), "=d" (regs->edx)
      : "a" (eax), "c" (ecx)
      : "memory"
   );
}

static INLINE uint32
__GET_EAX_FROM_CPUID(int eax) // IN
{
   uint32 ebx;

__asm__ __volatile__(
      VM_CPUID_BLOCK
      : "=a" (eax), VM_EBX_OUT(ebx)
      : "a" (eax)
      : "memory", "%ecx", "%edx"
   );

return eax;
}

static INLINE uint32
__GET_EBX_FROM_CPUID(int eax) // IN
{
   uint32 ebx;

__asm__ __volatile__(
      VM_CPUID_BLOCK
      : "=a" (eax), VM_EBX_OUT(ebx)
      : "a" (eax)
      : "memory", "%ecx", "%edx"
   );

return ebx;
}

static INLINE uint32
__GET_ECX_FROM_CPUID(int eax) // IN
{
   uint32 ecx;
   uint32 ebx;

__asm__ __volatile__(
      VM_CPUID_BLOCK
      : "=a" (eax), VM_EBX_OUT(ebx), "=c" (ecx)
      : "a" (eax)
      : "memory", "%edx"
   );

return ecx;
}

static INLINE uint32
__GET_EDX_FROM_CPUID(int eax) // IN
{
   uint32 edx;
   uint32 ebx;

__asm__ __volatile__(
      VM_CPUID_BLOCK
      : "=a" (eax), VM_EBX_OUT(ebx), "=d" (edx)
      : "a" (eax)
      : "memory", "%ecx"
   );

return edx;
}

static INLINE uint32
__GET_EAX_FROM_CPUID4(int ecx) // IN
{
   uint32 eax;
   uint32 ebx;

__asm__ __volatile__(
      VM_CPUID_BLOCK
      : "=a" (eax), VM_EBX_OUT(ebx), "=c" (ecx)
      : "a" (4), "c" (ecx)
      : "memory", "%edx"
   );

return eax;
}

#undef VM_CPUID_BLOCK
#undef VM_EBX_OUT

#elif defined(_MSC_VER) // } {

static INLINE void
__GET_CPUID(int input, CPUIDRegs *regs)
{
#ifdef VM_X86_64
   __cpuid((int *)regs, input);
#else
   __asm push esi
   __asm push ebx
   __asm push ecx
   __asm push edx

__asm mov  eax, input
   __asm mov  esi, regs
   __asm _emit 0x0f __asm _emit 0xa2
   __asm mov 0x0[esi], eax
   __asm mov 0x4[esi], ebx
   __asm mov 0x8[esi], ecx
   __asm mov 0xC[esi], edx

__asm pop edx
   __asm pop ecx
   __asm pop ebx
   __asm pop esi
#endif
}

#ifdef VM_X86_64

#if _MSC_VER >= 1500

/*
 * __cpuidex has been supported since VS2008
 */

static INLINE void
__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
{
   __cpuidex((int *)regs, inputEax, inputEcx);
}

#else // _MSC_VER >= 1500

/*
 * No inline assembly in Win64. Implemented in bora/lib/misc in
 * cpuidMasm64.asm.
 */

extern void
__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs);
#endif // _MSC_VER >= 1500

#else // VM_X86_64

static INLINE void
__GET_CPUID2(int inputEax, int inputEcx, CPUIDRegs *regs)
{
   __asm push esi
   __asm push ebx
   __asm push ecx
   __asm push edx

__asm mov  eax, inputEax
   __asm mov  ecx, inputEcx
   __asm mov  esi, regs
   __asm _emit 0x0f __asm _emit 0xa2
   __asm mov 0x0[esi], eax
   __asm mov 0x4[esi], ebx
   __asm mov 0x8[esi], ecx
   __asm mov 0xC[esi], edx

__asm pop edx
   __asm pop ecx
   __asm pop ebx
   __asm pop esi
}
#endif

static INLINE uint32
__GET_EAX_FROM_CPUID(int input)
{
#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.eax;
#else
   uint32 output;

//NOT_TESTED();
   __asm push ebx
   __asm push ecx
   __asm push edx

__asm mov  eax, input
   __asm _emit 0x0f __asm _emit 0xa2
   __asm mov  output, eax

__asm pop edx
   __asm pop ecx
   __asm pop ebx

return output;
#endif
}

static INLINE uint32
__GET_EBX_FROM_CPUID(int input)
{
#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.ebx;
#else
   uint32 output;

//NOT_TESTED();
   __asm push ebx
   __asm push ecx
   __asm push edx

__asm mov  eax, input
   __asm _emit 0x0f __asm _emit 0xa2
   __asm mov  output, ebx

__asm pop edx
   __asm pop ecx
   __asm pop ebx

return output;
#endif
}

static INLINE uint32
__GET_ECX_FROM_CPUID(int input)
{
#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.ecx;
#else
   uint32 output;

//NOT_TESTED();
   __asm push ebx
   __asm push ecx
   __asm push edx

__asm mov  eax, input
   __asm _emit 0x0f __asm _emit 0xa2
   __asm mov  output, ecx

__asm pop edx
   __asm pop ecx
   __asm pop ebx

return output;
#endif
}

static INLINE uint32
__GET_EDX_FROM_CPUID(int input)
{
#ifdef VM_X86_64
   CPUIDRegs regs;
   __cpuid((int *)&regs, input);
   return regs.edx;
#else
   uint32 output;

//NOT_TESTED();
   __asm push ebx
   __asm push ecx
   __asm push edx

__asm mov  eax, input
   __asm _emit 0x0f __asm _emit 0xa2
   __asm mov  output, edx

__asm pop edx
   __asm pop ecx
   __asm pop ebx

return output;
#endif
}

#ifdef VM_X86_64

/*
 * No inline assembly in Win64. Implemented in bora/lib/misc in
 * cpuidMasm64.asm.
 */

extern uint32
__GET_EAX_FROM_CPUID4(int inputEcx);

#else // VM_X86_64

static INLINE uint32
__GET_EAX_FROM_CPUID4(int inputEcx)
{
   uint32 output;

//NOT_TESTED();
   __asm push ebx
   __asm push ecx
   __asm push edx

__asm mov  eax, 4
   __asm mov  ecx, inputEcx
   __asm _emit 0x0f __asm _emit 0xa2
   __asm mov  output, eax

__asm pop edx
   __asm pop ecx
   __asm pop ebx

return output;
}

#endif // VM_X86_64

#else // }
#error
#endif

#define CPUID_FOR_SIDE_EFFECTS() ((void)__GET_EAX_FROM_CPUID(0))

/* The first parameter is used as an rvalue and then as an lvalue. */
#define GET_CPUID(_ax, _bx, _cx, _dx) { \
   CPUIDRegs regs;                      \
   __GET_CPUID(_ax, &regs);             \
   _ax = regs.eax;                      \
   _bx = regs.ebx;                      \
   _cx = regs.ecx;                      \
   _dx = regs.edx;                      \
}

#define GET_CPUID2(_ax, _bx, _cx, _dx) {\
   CPUIDRegs regs;                      \
   __GET_CPUID2(_ax, _cx, &regs);       \
   _ax = regs.eax;                      \
   _bx = regs.ebx;                      \
   _cx = regs.ecx;                      \
   _dx = regs.edx;                      \
}

#ifndef _VMCI_CALL_DEFS_H_
#define _VMCI_CALL_DEFS_H_

#define INCLUDE_ALLOW_USERLEVEL

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMMON
#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_VMKMOD
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#include "includeCheck.h"

#include "vm_basic_types.h"
#include "vmci_defs.h"

#if defined __cplusplus
extern "C" {
#endif

/*
 * All structs here are an integral size of their largest member, ie. a struct 
 * with at least one 8-byte member will have a size that is an integral of 8.
 * A struct which has a largest member of size 4 will have a size that is an
 * integral of 4. This is because Windows CL enforces this rule. 32 bit gcc 
 * doesn't e.g. 32 bit gcc can misalign an 8 byte member if it is preceeded by
 * a 4 byte member. 
 */

/*
 * Base struct for vmci datagrams.
 */

typedef struct VMCIDatagram {
   VMCIHandle dst;
   VMCIHandle src;
   uint64     payloadSize;
} VMCIDatagram;

/*
 * Second flag is for creating a well-known handle instead of a per context
 * handle.  Next flag is for deferring datagram delivery, so that the
 * datagram callback is invoked in a delayed context (not interrupt context).
 */
#define VMCI_FLAG_DG_NONE          0
#define VMCI_FLAG_WELLKNOWN_DG_HND 0x1
#define VMCI_FLAG_ANYCID_DG_HND    0x2
#define VMCI_FLAG_DG_DELAYED_CB    0x4

/* Event callback should fire in a delayed context (not interrupt context.) */
#define VMCI_FLAG_EVENT_NONE       0
#define VMCI_FLAG_EVENT_DELAYED_CB 0x1

/* 
 * Maximum supported size of a VMCI datagram for routable datagrams.
 * Datagrams going to the hypervisor are allowed to be larger.
 */
#define VMCI_MAX_DG_SIZE (17 * 4096)
#define VMCI_MAX_DG_PAYLOAD_SIZE (VMCI_MAX_DG_SIZE - sizeof(VMCIDatagram))
#define VMCI_DG_PAYLOAD(_dg) (void *)((char *)(_dg) + sizeof(VMCIDatagram))
#define VMCI_DG_HEADERSIZE sizeof(VMCIDatagram)
#define VMCI_DG_SIZE(_dg) (VMCI_DG_HEADERSIZE + (size_t)(_dg)->payloadSize)
#define VMCI_DG_SIZE_ALIGNED(_dg) ((VMCI_DG_SIZE(_dg) + 7) & (size_t)~7)
#define VMCI_MAX_DATAGRAM_QUEUE_SIZE (VMCI_MAX_DG_SIZE * 2)

/*
 * We allow at least 1024 more event datagrams from the hypervisor past the
 * normally allowed datagrams pending for a given context.  We define this
 * limit on event datagrams from the hypervisor to guard against DoS attack
 * from a malicious VM which could repeatedly attach to and detach from a queue
 * pair, causing events to be queued at the destination VM.  However, the rate
 * at which such events can be generated is small since it requires a VM exit
 * and handling of queue pair attach/detach call at the hypervisor.  Event
 * datagrams may be queued up at the destination VM if it has interrupts
 * disabled or if it is not draining events for some other reason.  1024
 * datagrams is a grossly conservative estimate of the time for which
 * interrupts may be disabled in the destination VM, but at the same time does
 * not exacerbate the memory pressure problem on the host by much (size of each
 * event datagram is small).
 */
#define VMCI_MAX_DATAGRAM_AND_EVENT_QUEUE_SIZE \
   (VMCI_MAX_DATAGRAM_QUEUE_SIZE + \
    1024 * (sizeof(VMCIDatagram) + sizeof(VMCIEventData_Max)))

/*
 * Struct for sending VMCI_DATAGRAM_REQUEST_MAP and
 * VMCI_DATAGRAM_REMOVE_MAP datagrams. Struct size is 32 bytes. All
 * fields in struct are aligned to their natural alignment. These
 * datagrams are obsoleted by the removal of VM to VM communication.
 */
typedef struct VMCIDatagramWellKnownMapMsg {
   VMCIDatagram hdr;
   VMCIId       wellKnownID;
   uint32       _pad;
} VMCIDatagramWellKnownMapMsg;

/*
 * Struct used for querying, via VMCI_RESOURCES_QUERY, the availability of 
 * hypervisor resources. 
 * Struct size is 16 bytes. All fields in struct are aligned to their natural
 * alignment.
 */
typedef struct VMCIResourcesQueryHdr {
   VMCIDatagram hdr;
   uint32       numResources;
   uint32       _padding;
} VMCIResourcesQueryHdr;

/*
 * Convenience struct for negotiating vectors. Must match layout of
 * VMCIResourceQueryHdr minus the VMCIDatagram header.
 */
typedef struct VMCIResourcesQueryMsg {
   uint32        numResources;
   uint32        _padding;
   VMCI_Resource resources[1];
} VMCIResourcesQueryMsg;

/* 
 * The maximum number of resources that can be queried using
 * VMCI_RESOURCE_QUERY is 31, as the result is encoded in the lower 31
 * bits of a positive return value. Negative values are reserved for
 * errors.
 */
#define VMCI_RESOURCE_QUERY_MAX_NUM 31

/* Maximum size for the VMCI_RESOURCE_QUERY request. */
#define VMCI_RESOURCE_QUERY_MAX_SIZE sizeof(VMCIResourcesQueryHdr) \
      + VMCI_RESOURCE_QUERY_MAX_NUM * sizeof(VMCI_Resource)

/* 
 * Struct used for setting the notification bitmap.  All fields in
 * struct are aligned to their natural alignment.
 */
typedef struct VMCINotifyBitmapSetMsg {
   VMCIDatagram hdr;
   PPN32        bitmapPPN;
   uint32       _pad;
} VMCINotifyBitmapSetMsg;

/*
 * Struct used for linking a doorbell handle with an index in the
 * notify bitmap. All fields in struct are aligned to their natural
 * alignment.
 */
typedef struct VMCIDoorbellLinkMsg {
   VMCIDatagram hdr;
   VMCIHandle   handle;
   uint64       notifyIdx;
} VMCIDoorbellLinkMsg;

/* 
 * Struct used for unlinking a doorbell handle from an index in the
 * notify bitmap. All fields in struct are aligned to their natural
 * alignment.
 */
typedef struct VMCIDoorbellUnlinkMsg {
   VMCIDatagram hdr;
   VMCIHandle   handle;
} VMCIDoorbellUnlinkMsg;

/* 
 * Struct used for generating a notification on a doorbell handle. All
 * fields in struct are aligned to their natural alignment.
 */
typedef struct VMCIDoorbellNotifyMsg {
   VMCIDatagram hdr;
   VMCIHandle   handle;
} VMCIDoorbellNotifyMsg;

/* 
 * This struct is used to contain data for events.  Size of this struct is a
 * multiple of 8 bytes, and all fields are aligned to their natural alignment.
 */
typedef struct VMCI_EventData {
   VMCI_Event event; /* 4 bytes. */
   uint32     _pad;
   /*
    * Event payload is put here.
    */
} VMCI_EventData;

/* Callback needed for correctly waiting on events. */

typedef int
(*VMCIDatagramRecvCB)(void *clientData,   // IN: client data for handler
                      VMCIDatagram *msg); // IN:

/*
 * We use the following inline function to access the payload data associated
 * with an event data.
 */

static INLINE void *
VMCIEventDataPayload(VMCI_EventData *evData) // IN:
{
   return (void *)((char *)evData + sizeof *evData);
}

/*
 * Define the different VMCI_EVENT payload data types here.  All structs must
 * be a multiple of 8 bytes, and fields must be aligned to their natural
 * alignment.
 */
typedef struct VMCIEventPayload_Context {
   VMCIId contextID; /* 4 bytes. */
   uint32 _pad;
} VMCIEventPayload_Context;

typedef struct VMCIEventPayload_QP {
   VMCIHandle handle; /* QueuePair handle. */
   VMCIId     peerId; /* Context id of attaching/detaching VM. */
   uint32     _pad;
} VMCIEventPayload_QP;

/*
 * We define the following struct to get the size of the maximum event data
 * the hypervisor may send to the guest.  If adding a new event payload type
 * above, add it to the following struct too (inside the union).
 */
typedef struct VMCIEventData_Max {
   VMCI_EventData eventData;
   union {
      VMCIEventPayload_Context contextPayload;
      VMCIEventPayload_QP      qpPayload;
   } evDataPayload;
} VMCIEventData_Max;

/* 
 * Struct used for VMCI_EVENT_SUBSCRIBE/UNSUBSCRIBE and VMCI_EVENT_HANDLER 
 * messages.  Struct size is 32 bytes.  All fields in struct are aligned to
 * their natural alignment.
 */
typedef struct VMCIEventMsg {
   VMCIDatagram   hdr;
   VMCI_EventData eventData; /* Has event type and payload. */
   /*
    * Payload gets put here.
    */
} VMCIEventMsg;

/*
 * We use the following inline function to access the payload data associated
 * with an event message.
 */

static INLINE void *
VMCIEventMsgPayload(VMCIEventMsg *eMsg) // IN:
{
   return VMCIEventDataPayload(&eMsg->eventData);
}

/* Flags for VMCI QueuePair API. */
#define VMCI_QPFLAG_ATTACH_ONLY 0x1 /* Fail alloc if QP not created by peer. */
#define VMCI_QPFLAG_LOCAL       0x2 /* Only allow attaches from local context. */
#define VMCI_QPFLAG_NONBLOCK    0x4 /* Host won't block when guest is quiesced. */
/* For asymmetric queuepairs, update as new flags are added. */
#define VMCI_QP_ASYMM           VMCI_QPFLAG_NONBLOCK
#define VMCI_QP_ASYMM_PEER      (VMCI_QPFLAG_ATTACH_ONLY | VMCI_QP_ASYMM)
/* Update the following (bitwise OR flags) while adding new flags. */
#define VMCI_QP_ALL_FLAGS       (VMCI_QPFLAG_ATTACH_ONLY | VMCI_QPFLAG_LOCAL | \
                                 VMCI_QPFLAG_NONBLOCK)

/*
 * Structs used for QueuePair alloc and detach messages.  We align fields of
 * these structs to 64bit boundaries.
 */

typedef struct VMCIQueuePairAllocMsg {
   VMCIDatagram   hdr;
   VMCIHandle     handle;
   VMCIId         peer; /* 32bit field. */
   uint32         flags;
   uint64         produceSize;
   uint64         consumeSize;
   uint64         numPPNs;
   /* List of PPNs placed here. */
} VMCIQueuePairAllocMsg;

typedef struct VMCIQueuePairDetachMsg {
   VMCIDatagram  hdr;
   VMCIHandle    handle;
} VMCIQueuePairDetachMsg;

#if defined __cplusplus
} // extern "C"
#endif

#endif // _VMCI_CALL_DEFS_H_
�����������������������������������������������������vmxnet3-only/shared/compat_interrupt.h��������������������������������������������������������������0000444�0000000�0000000�00000003573�13432725347�017175� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2003 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef __COMPAT_INTERRUPT_H__
#   define __COMPAT_INTERRUPT_H__

#include <linux/interrupt.h>

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 69)
/*
 * We cannot just define irqreturn_t, as some 2.4.x kernels have
 * typedef void irqreturn_t; for "increasing" backward compatibility.
 */
typedef void compat_irqreturn_t;
#define COMPAT_IRQ_NONE
#define COMPAT_IRQ_HANDLED
#define COMPAT_IRQ_RETVAL(x)
#else
typedef irqreturn_t compat_irqreturn_t;
#define COMPAT_IRQ_NONE		IRQ_NONE
#define COMPAT_IRQ_HANDLED	IRQ_HANDLED
#define COMPAT_IRQ_RETVAL(x)	IRQ_RETVAL(x)
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 18)
#define COMPAT_IRQF_DISABLED    SA_INTERRUPT
#define COMPAT_IRQF_SHARED      SA_SHIRQ
#else
#define COMPAT_IRQF_DISABLED    IRQF_DISABLED
#define COMPAT_IRQF_SHARED      IRQF_SHARED
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 20)
#define COMPAT_IRQ_HANDLER_ARGS(irq, devp) (int irq, void *devp, struct pt_regs *regs)
#else
#define COMPAT_IRQ_HANDLER_ARGS(irq, devp) (int irq, void *devp)
#endif

#endif /* __COMPAT_INTERRUPT_H__ */
�������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/COPYING��������������������������������������������������������������������������������0000444�0000000�0000000�00000043103�13432725341�013175� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������		    GNU GENERAL PUBLIC LICENSE
		       Version 2, June 1991

Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 Everyone is permitted to copy and distribute verbatim copies
 of this license document, but changing it is not allowed.

Preamble

The licenses for most software are designed to take away your
freedom to share and change it.  By contrast, the GNU General Public
License is intended to guarantee your freedom to share and change free
software--to make sure the software is free for all its users.  This
General Public License applies to most of the Free Software
Foundation's software and to any other program whose authors commit to
using it.  (Some other Free Software Foundation software is covered by
the GNU Lesser General Public License instead.)  You can apply it to
your programs, too.

When we speak of free software, we are referring to freedom, not
price.  Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
this service if you wish), that you receive source code or can get it
if you want it, that you can change the software or use pieces of it
in new free programs; and that you know you can do these things.

To protect your rights, we need to make restrictions that forbid
anyone to deny you these rights or to ask you to surrender the rights.
These restrictions translate to certain responsibilities for you if you
distribute copies of the software, or if you modify it.

For example, if you distribute copies of such a program, whether
gratis or for a fee, you must give the recipients all the rights that
you have.  You must make sure that they, too, receive or can get the
source code.  And you must show them these terms so they know their
rights.

We protect your rights with two steps: (1) copyright the software, and
(2) offer you this license which gives you legal permission to copy,
distribute and/or modify the software.

Also, for each author's protection and ours, we want to make certain
that everyone understands that there is no warranty for this free
software.  If the software is modified by someone else and passed on, we
want its recipients to know that what they have is not the original, so
that any problems introduced by others will not reflect on the original
authors' reputations.

Finally, any free program is threatened constantly by software
patents.  We wish to avoid the danger that redistributors of a free
program will individually obtain patent licenses, in effect making the
program proprietary.  To prevent this, we have made it clear that any
patent must be licensed for everyone's free use or not licensed at all.

The precise terms and conditions for copying, distribution and
modification follow.

GNU GENERAL PUBLIC LICENSE
   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION

0. This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License.  The "Program", below,
refers to any such program or work, and a "work based on the Program"
means either the Program or any derivative work under copyright law:
that is to say, a work containing the Program or a portion of it,
either verbatim or with modifications and/or translated into another
language.  (Hereinafter, translation is included without limitation in
the term "modification".)  Each licensee is addressed as "you".

Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope.  The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.

1. You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License
along with the Program.

You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.

2. You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:

a) You must cause the modified files to carry prominent notices
    stating that you changed the files and the date of any change.

b) You must cause any work that you distribute or publish, that in
    whole or in part contains or is derived from the Program or any
    part thereof, to be licensed as a whole at no charge to all third
    parties under the terms of this License.

c) If the modified program normally reads commands interactively
    when run, you must cause it, when started running for such
    interactive use in the most ordinary way, to print or display an
    announcement including an appropriate copyright notice and a
    notice that there is no warranty (or else, saying that you provide
    a warranty) and that users may redistribute the program under
    these conditions, and telling the user how to view a copy of this
    License.  (Exception: if the Program itself is interactive but
    does not normally print such an announcement, your work based on
    the Program is not required to print an announcement.)

These requirements apply to the modified work as a whole.  If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works.  But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.

Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.

In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.

3. You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:

a) Accompany it with the complete corresponding machine-readable
    source code, which must be distributed under the terms of Sections
    1 and 2 above on a medium customarily used for software interchange; or,

b) Accompany it with a written offer, valid for at least three
    years, to give any third party, for a charge no more than your
    cost of physically performing source distribution, a complete
    machine-readable copy of the corresponding source code, to be
    distributed under the terms of Sections 1 and 2 above on a medium
    customarily used for software interchange; or,

c) Accompany it with the information you received as to the offer
    to distribute corresponding source code.  (This alternative is
    allowed only for noncommercial distribution and only if you
    received the program in object code or executable form with such
    an offer, in accord with Subsection b above.)

The source code for a work means the preferred form of the work for
making modifications to it.  For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable.  However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.

If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.

4. You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License.  Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.

5. You are not required to accept this License, since you have not
signed it.  However, nothing else grants you permission to modify or
distribute the Program or its derivative works.  These actions are
prohibited by law if you do not accept this License.  Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.

6. Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions.  You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.

7. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License.  If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all.  For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.

If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.

It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices.  Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.

This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.

8. If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded.  In such case, this License incorporates
the limitation as if written in the body of this License.

9. The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time.  Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.

Each version is given a distinguishing version number.  If the Program
specifies a version number of this License which applies to it and "any
later version", you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation.  If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.

10. If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission.  For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this.  Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.

NO WARRANTY

11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.

12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.

END OF TERMS AND CONDITIONS

How to Apply These Terms to Your New Programs

If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.

To do so, attach the following notices to the program.  It is safest
to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.

<one line to give the program's name and a brief idea of what it does.>
    Copyright (C) <year>  <name of author>

This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

You should have received a copy of the GNU General Public License along
    with this program; if not, write to the Free Software Foundation, Inc.,
    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

Also add information on how to contact you by electronic and paper mail.

If the program is interactive, make it output a short notice like this
when it starts in an interactive mode:

Gnomovision version 69, Copyright (C) year name of author
    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
    This is free software, and you are welcome to redistribute it
    under certain conditions; type `show c' for details.

The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License.  Of course, the commands you use may
be called something other than `show w' and `show c'; they could even be
mouse-clicks or menu items--whatever suits your program.

You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the program, if
necessary.  Here is a sample; alter the names:

Yoyodyne, Inc., hereby disclaims all copyright interest in the program
  `Gnomovision' (which makes passes at compilers) written by James Hacker.

<signature of Ty Coon>, 1 April 1989
  Ty Coon, President of Vice

This General Public License does not permit incorporating your program into
proprietary programs.  If your program is a subroutine library, you may
consider it more useful to permit linking proprietary applications with the
library.  If this is what you want to do, use the GNU Lesser General
Public License instead of this License.
�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/net_sg.h�������������������������������������������������������������������������������0000444�0000000�0000000�00000003760�13432725350�013577� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2000 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * net_sg.h --
 *
 *	Network packet scatter gather structure.
 */

#ifndef _NET_SG_H
#define _NET_SG_H

#define INCLUDE_ALLOW_USERLEVEL

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#include "includeCheck.h"

#define NET_SG_DEFAULT_LENGTH	16

/*
 * A single scatter-gather element for a network packet.
 * The address is split into low and high to save space.
 * If we make it 64 bits then Windows pads things out such that
 * we lose a lot of space for each scatter gather array.
 * This adds up when you have embedded scatter-gather 
 * arrays for transmit and receive ring buffers.
 */
typedef struct NetSG_Elem {
   uint32 	addrLow;
   uint16	addrHi;
   uint16	length;
} NetSG_Elem;

typedef enum NetSG_AddrType {
   NET_SG_MACH_ADDR,
   NET_SG_PHYS_ADDR,
   NET_SG_VIRT_ADDR,
} NetSG_AddrType;

typedef struct NetSG_Array {
   uint16	addrType;
   uint16	length;
   NetSG_Elem	sg[NET_SG_DEFAULT_LENGTH];
} NetSG_Array;

#define NET_SG_MAKE_PA(elem)                 (PA)QWORD(elem.addrHi, elem.addrLow)
#define NET_SG_MAKE_PTR(elem) (char *)(uintptr_t)QWORD(elem.addrHi, elem.addrLow)

#endif
����������������vmxnet3-only/vmxnet3_shm.c��������������������������������������������������������������������������0000444�0000000�0000000�00000104625�13432725331�014567� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2009 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmxnet3_shm.c --
 *
 *    Shared memory infrastructure for VMXNET3 linux driver. Used by the
 *    VMXNET3 driver to back its rings with memory from a shared memory
 *    pool that is shared with user space.
 */
#include "driver-config.h"

#include "vmxnet3_int.h"
#include "vmnet_def.h"
#include "vm_device_version.h"
#include "vmxnet3_shm.h"

static int
vmxnet3_shm_consume_user_tx_queue(struct vmxnet3_shm_pool *shm);

int
vmxnet3_shm_tq_xmit(struct sk_buff *skb,
		struct vmxnet3_tx_queue *tq,
		struct vmxnet3_adapter *adapter,
		struct net_device *netdev);

/*
 *----------------------------------------------------------------------------
 *
 * kernel_rx_idx --
 *
 * Result:
 *    Kernel's current shared memory RX ring index
 *
 * Side-effects:
 *    None
 *
 *----------------------------------------------------------------------------
 */

static inline u16
kernel_rx_idx(const struct vmxnet3_shm_pool *shm)
{
	return shm->ctl.ptr->kernel_rxi;
}

/*
 *----------------------------------------------------------------------------
 *
 * inc_kernel_rx_idx --
 *
 * Result:
 *    None
 *
 * Side-effects:
 *    Increment the kernel's shared memory RX ring index
 *
 *----------------------------------------------------------------------------
 */

static inline void
inc_kernel_rx_idx(const struct vmxnet3_shm_pool *shm)
{
	shm->ctl.ptr->kernel_rxi = (shm->ctl.ptr->kernel_rxi + 1) % SHM_RX_RING_SIZE;
}

static inline u16
kernel_tx_idx(const struct vmxnet3_shm_pool *shm)
{
	return shm->ctl.ptr->kernel_txi;
}

/*
 *----------------------------------------------------------------------------
 *
 * inc_kernel_tx_idx --
 *
 * Result:
 *    None
 *
 * Side-effects:
 *    Increment the kernel's shared memory TX ring index
 *
 *----------------------------------------------------------------------------
 */

static inline void
inc_kernel_tx_idx(const struct vmxnet3_shm_pool *shm)
{
	shm->ctl.ptr->kernel_txi = (shm->ctl.ptr->kernel_txi + 1) % SHM_TX_RING_SIZE;
}

/*
 *----------------------------------------------------------------------------
 *
 * user_rx_idx --
 *
 * Result:
 *    Users's current shared memory RX ring index
 *
 * Side-effects:
 *    None
 *
 *----------------------------------------------------------------------------
 */

static inline u16
user_rx_idx(const struct vmxnet3_shm_pool *shm)
{
	return shm->ctl.ptr->user_rxi;
}

/*
 *----------------------------------------------------------------------------
 *
 * kernel_rx_entry --
 *
 * Result:
 *    Kernel's current shared memory RX ring entry
 *
 * Side-effects:
 *    None
 *
 *----------------------------------------------------------------------------
 */

static inline struct vmxnet3_shm_ringentry *
kernel_rx_entry(const struct vmxnet3_shm_pool *shm)
{
	return &shm->ctl.ptr->rx_ring[kernel_rx_idx(shm)];
}

/*
 *----------------------------------------------------------------------------
 *
 * kernel_tx_entry --
 *
 * Result:
 *    Kernel's current shared memory TX ring entry
 *
 * Side-effects:
 *    None
 *
 *----------------------------------------------------------------------------
 */

static inline struct vmxnet3_shm_ringentry *
kernel_tx_entry(const struct vmxnet3_shm_pool *shm)
{
	return &shm->ctl.ptr->tx_ring[kernel_tx_idx(shm)];
}

/*
 *----------------------------------------------------------------------------
 *
 * user_rx_entry --
 *
 * Used by vmxnet3_shm_chardev_poll
 *
 * Result:
 *    User's current shared memory RX ring entry
 *
 * Side-effects:
 *    None
 *
 *----------------------------------------------------------------------------
 */

static inline struct vmxnet3_shm_ringentry *
user_rx_entry(const struct vmxnet3_shm_pool *shm)
{
	return &shm->ctl.ptr->rx_ring[user_rx_idx(shm)];
}

/* kobject type */
static void
vmxnet3_shm_pool_release(struct kobject *kobj);

static const struct kobj_type vmxnet3_shm_pool_type = {
	.release = vmxnet3_shm_pool_release
};

/* vm operations */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
static int
vmxnet3_shm_chardev_fault(struct vm_area_struct *vma,
		struct vm_fault *vmf);

static struct vm_operations_struct vmxnet3_shm_vm_ops = {
	.fault = vmxnet3_shm_chardev_fault,
};
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 1)
static struct page *
vmxnet3_shm_chardev_nopage(struct vm_area_struct *vma,
		unsigned long address,
		int *type);

static struct vm_operations_struct vmxnet3_shm_vm_ops = {
	.nopage = vmxnet3_shm_chardev_nopage,
};
#else
static struct page *
vmxnet3_shm_chardev_nopage(struct vm_area_struct *vma,
		unsigned long address,
		int unused);

static struct vm_operations_struct vmxnet3_shm_vm_ops = {
	.nopage = vmxnet3_shm_chardev_nopage,
};
#endif

/* file operations */
static int vmxnet3_shm_chardev_mmap(struct file *filp,
		struct vm_area_struct *vma);

static int vmxnet3_shm_chardev_open(struct inode * inode,
		struct file * filp);

static int vmxnet3_shm_chardev_release(struct inode * inode,
		struct file * filp);

static unsigned int vmxnet3_shm_chardev_poll(struct file *filp,
		poll_table *wait);

static long vmxnet3_shm_chardev_ioctl(struct file *filp,
		unsigned int cmd,
		unsigned long arg);

#ifndef HAVE_UNLOCKED_IOCTL
static int vmxnet3_shm_chardev_old_ioctl(struct inode *inode,
		struct file *filp,
		unsigned int cmd,
		unsigned long arg);
#endif

static struct file_operations shm_fops = {
	.owner = THIS_MODULE,
	.mmap = vmxnet3_shm_chardev_mmap,
	.open = vmxnet3_shm_chardev_open,
	.release = vmxnet3_shm_chardev_release,
	.poll = vmxnet3_shm_chardev_poll,
#ifdef HAVE_UNLOCKED_IOCTL
	.unlocked_ioctl = vmxnet3_shm_chardev_ioctl,
#ifdef CONFIG_COMPAT
	.compat_ioctl = vmxnet3_shm_chardev_ioctl,
#endif
#else
	.ioctl = vmxnet3_shm_chardev_old_ioctl,
#endif
};

static LIST_HEAD(vmxnet3_shm_list);
static DEFINE_SPINLOCK(vmxnet3_shm_list_lock);

/* vmxnet3_shm_pool kobject */

/* Lifecycle */

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 25)
#define compat_kobject_init(kobj, ktype) { kobject_init(kobj, (struct kobj_type *) ktype); }
#else
#define compat_kobject_init(kobj, _ktype) {  \
	(kobj)->ktype = (struct kobj_type *) _ktype; \
	kobject_init(kobj); \
}
#endif

/*
 *----------------------------------------------------------------------------
 *
 * vmxnet3_shm_init_allocator --
 *
 * Zero all shared memory data pages and fill the allocator with them.
 *
 * Result:
 *    None
 *
 *----------------------------------------------------------------------------
 */

static void
vmxnet3_shm_init_allocator(struct vmxnet3_shm_pool *shm)
{
	int i;

shm->allocator.count = 0;
	for (i = 1; i < shm->data.num_pages; i++) {
		struct page *page = VMXNET3_SHM_IDX2PAGE(shm, i);
		void *virt = kmap(page);
		memset(virt, 0, PAGE_SIZE);
		kunmap(page);

shm->allocator.stack[shm->allocator.count++] = i;

BUG_ON(i == SHM_INVALID_IDX);
	}
	BUG_ON(shm->allocator.count > shm->data.num_pages);
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_pool_reset --
 *
 *    Clean up after userspace has closed the device
 *
 * Results:
 *    None.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static void
vmxnet3_shm_pool_reset(struct vmxnet3_shm_pool *shm)
{
	int err = 0;
	printk(KERN_INFO "resetting shm pool\n");

/*
	 * Reset_work may be in the middle of resetting the device, wait for its
	 * completion.
	 */
	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &shm->adapter->state))
		compat_msleep(1);

if (netif_running(shm->adapter->netdev))
		vmxnet3_quiesce_dev(shm->adapter);

vmxnet3_shm_init_allocator(shm);

if (netif_running(shm->adapter->netdev))
		err = vmxnet3_activate_dev(shm->adapter);

memset(shm->ctl.ptr, 0, PAGE_SIZE);

clear_bit(VMXNET3_STATE_BIT_RESETTING, &shm->adapter->state);

if (err)
		vmxnet3_force_close(shm->adapter);
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_pool_create --
 *
 *    Allocate and initialize shared memory pool. Allocates the data and
 *    control pages, resets them to zero, initializes locks, registers the
 *    character device, etc. Creates virtual address mappings for the pool,
 *    but does not set up DMA yet.
 *
 * Results:
 *    The new shared memory pool object, or NULL on failure.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

struct vmxnet3_shm_pool *
vmxnet3_shm_pool_create(struct vmxnet3_adapter *adapter,
		char *name, int pool_size)
{
	int i;
	unsigned long flags;
	struct vmxnet3_shm_pool *shm;
	struct vmxnet3_shm_ctl *ctl_ptr;
	struct page *ctl_page;
	int shm_size = sizeof(*shm) +
		pool_size * sizeof(uint16) +
		pool_size * sizeof(struct vmxnet3_shm_mapped_page);

/* Allocate shm_pool kobject */
	shm = vmalloc(shm_size);
	if (shm == NULL)
		goto fail_shm;

memset(shm, 0, shm_size);
	shm->data.pages = (struct vmxnet3_shm_mapped_page*)(shm + 1);
	shm->allocator.stack = (uint16*)(shm->data.pages + pool_size);

compat_kobject_init(&shm->kobj, &vmxnet3_shm_pool_type);
	/* shm->kobj.ktype = &vmxnet3_shm_pool_type; */
	/* kobject_init(&shm->kobj); */
	snprintf(shm->name, sizeof(shm->name), "vmxnet_%s_shm", name);
	kobject_set_name(&shm->kobj, shm->name);
	shm->adapter = adapter;

/* Allocate data pages */
	shm->data.num_pages = pool_size;
	for (i = 1; i < shm->data.num_pages; i++) {
		struct page *page = alloc_page(GFP_KERNEL);
		if (page == NULL)
			goto fail_data;

VMXNET3_SHM_SET_IDX2PAGE(shm, i, page);

BUG_ON(i == SHM_INVALID_IDX);
	}

/* Allocate control page */
	ctl_page = alloc_page(GFP_KERNEL);
	if (ctl_page == NULL)
		goto fail_ctl;

ctl_ptr = (void*)kmap(ctl_page);
	shm->ctl.pages[0] = ctl_page;
	shm->ctl.ptr = ctl_ptr;

/* Reset data and control pages */
	vmxnet3_shm_init_allocator(shm);
	memset(shm->ctl.ptr, 0, PAGE_SIZE);

/* Register char device */
	shm->misc_dev.minor = MISC_DYNAMIC_MINOR;
	shm->misc_dev.name = shm->name;
	shm->misc_dev.fops = &shm_fops;
	if (misc_register(&shm->misc_dev)) {
		printk(KERN_ERR "failed to register vmxnet3_shm character device\n");
		goto fail_cdev;
	}

/* Initialize locks */
	spin_lock_init(&shm->alloc_lock);
	spin_lock_init(&shm->tx_lock);
	spin_lock_init(&shm->rx_lock);
	init_waitqueue_head(&shm->rxq);

spin_lock_irqsave(&vmxnet3_shm_list_lock, flags);
	list_add(&shm->list, &vmxnet3_shm_list);
	spin_unlock_irqrestore(&vmxnet3_shm_list_lock, flags);

printk(KERN_INFO "created vmxnet shared memory pool %s\n", shm->name);

return shm;

fail_cdev:
	kunmap(ctl_page);
	__free_page(ctl_page);

fail_data:
fail_ctl:
	for (i = 0; i < shm->data.num_pages; i++) {
		if (VMXNET3_SHM_IDX2PAGE(shm, i) != NULL)
			__free_page(VMXNET3_SHM_IDX2PAGE(shm, i));
	}

vfree(shm);

fail_shm:
	return NULL;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_pool_release --
 *
 *    Release a shared memory pool.
 *
 * Results:
 *    None.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static void
vmxnet3_shm_pool_release(struct kobject *kobj)
{
	int i;
	unsigned long flags;
	struct vmxnet3_shm_pool *shm = container_of(kobj, struct vmxnet3_shm_pool, kobj);

spin_lock_irqsave(&vmxnet3_shm_list_lock, flags);
	list_del(&shm->list);
	spin_unlock_irqrestore(&vmxnet3_shm_list_lock, flags);

misc_deregister(&shm->misc_dev);

/* Free control pages */
	for (i = 0; i < SHM_CTL_SIZE; i++) {
		kunmap(shm->ctl.pages[i]);
		__free_page(shm->ctl.pages[i]);
	}

/* Free data pages */
	for (i = 1; i < shm->data.num_pages; i++)
		__free_page(VMXNET3_SHM_IDX2PAGE(shm, i));

printk(KERN_INFO "destroyed vmxnet shared memory pool %s\n", shm->name);

vfree(shm);
}

/* Shared memory pool management */

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_alloc_page --
 *
 *    Allocate a page from the shared memory area.
 *
 * Results:
 *    Index to page or SHM_INVALID_IDX on failure.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

uint16
vmxnet3_shm_alloc_page(struct vmxnet3_shm_pool *shm)
{
	uint16 idx;
	unsigned long flags;

spin_lock_irqsave(&shm->alloc_lock, flags);
	if (shm->allocator.count == 0) {
		idx = SHM_INVALID_IDX;
	} else {
		idx = shm->allocator.stack[--shm->allocator.count];
		BUG_ON(idx == SHM_INVALID_IDX);
	}
	spin_unlock_irqrestore(&shm->alloc_lock, flags);

return idx;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_free_page --
 *
 *    Free a page back to the shared memory area
 *
 * Results:
 *    None.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

void
vmxnet3_shm_free_page(struct vmxnet3_shm_pool *shm,
		uint16 idx)
{
	unsigned long flags;

spin_lock_irqsave(&shm->alloc_lock, flags);
	BUG_ON(shm->allocator.count >= shm->data.num_pages);
	shm->allocator.stack[shm->allocator.count++] = idx;
	spin_unlock_irqrestore(&shm->alloc_lock, flags);
}

/* Char device */

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_addr2idx --
 *
 *    Convert user space address into index into the shared memory pool.
 *
 * Results:
 *    None.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static inline unsigned long
vmxnet3_shm_addr2idx(struct vm_area_struct *vma,
		unsigned long address)
{
	return vma->vm_pgoff + ((address - vma->vm_start) >> PAGE_SHIFT);
}

#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 26)
/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_chardev_fault --
 *
 *    mmap fault handler. Called if the user space requests a page for
 *    which there is no shared memory mapping yet. We need to lookup
 *    the page we want to back the shared memory mapping with.
 *
 * Results:
 *    The page backing the user space address.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static int
vmxnet3_shm_chardev_fault(struct vm_area_struct *vma,
		struct vm_fault *vmf)
{
	struct vmxnet3_shm_pool *shm = vma->vm_private_data;
	unsigned long address = (unsigned long)vmf->virtual_address;
	unsigned long idx = vmxnet3_shm_addr2idx(vma, address);
	struct page *pageptr;

if (idx >= SHM_DATA_START && idx < SHM_DATA_START + shm->data.num_pages)
		pageptr = VMXNET3_SHM_IDX2PAGE(shm, idx - SHM_DATA_START);
	else if (idx >= SHM_CTL_START && idx < SHM_CTL_START + SHM_CTL_SIZE)
		pageptr = shm->ctl.pages[idx - SHM_CTL_START];
	else
		pageptr = NULL;

if (pageptr)
		get_page(pageptr);

vmf->page = pageptr;

return pageptr ? VM_FAULT_MINOR : VM_FAULT_ERROR;
}
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 1)

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_chardev_nopage --
 *
 *    mmap nopage handler. Called if the user space requests a page for
 *    which there is no shared memory mapping yet. We need to lookup
 *    the page we want to back the shared memory mapping with.
 *
 * Results:
 *    The page backing the user space address.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static struct page *
vmxnet3_shm_chardev_nopage(struct vm_area_struct *vma,
		unsigned long address,
		int *type)
{
	struct vmxnet3_shm_pool *shm = vma->vm_private_data;
	unsigned long idx = vmxnet3_shm_addr2idx(vma, address);
	struct page *pageptr;

if (pageptr)
		get_page(pageptr);

if (type)
		*type = pageptr ? VM_FAULT_MINOR : VM_FAULT_SIGBUS;

return pageptr;
}

#else
/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_chardev_nopage --
 *
 *    mmap nopage handler. Called if the user space requests a page for
 *    which there is no shared memory mapping yet. We need to lookup
 *    the page we want to back the shared memory mapping with.
 *
 * Results:
 *    The page backing the user space address.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static struct page *
vmxnet3_shm_chardev_nopage(struct vm_area_struct *vma,
		unsigned long address,
		int unused)
{
	struct vmxnet3_shm_pool *shm = vma->vm_private_data;
	unsigned long idx = vmxnet3_shm_addr2idx(vma, address);
	struct page *pageptr;

if (pageptr)
		get_page(pageptr);

return pageptr;
}
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_chardev_mmap --
 *
 *    Setup mmap.
 *
 * Results:
 *    Always 0.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */
static int
vmxnet3_shm_chardev_mmap(struct file *filp,
		struct vm_area_struct *vma)
{
	vma->vm_private_data = filp->private_data;
	vma->vm_ops = &vmxnet3_shm_vm_ops;
#if COMPAT_LINUX_VERSION_CHECK_LT(3, 7, 0)
	vma->vm_flags |= VM_RESERVED;
#else
	vma->vm_flags |= VM_IO;
#endif
	return 0;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_chardev_poll --
 *
 *    Poll called from user space. We consume the TX queue and then go to
 *    sleep until we get woken up by an interrupt.
 *
 * Results:
 *    Poll mask.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static unsigned int
vmxnet3_shm_chardev_poll(struct file *filp,
		poll_table *wait)
{
	struct vmxnet3_shm_pool *shm = filp->private_data;
	unsigned int mask = 0;
	unsigned long flags;
	struct vmxnet3_shm_ringentry *re;

/* consume TX queue */
	if (vmxnet3_shm_consume_user_tx_queue(shm) == -1) {
		/*
		 * The device has been closed, let the user space
		 * know there is activity, so that it gets a chance
		 * to read the channelBad flag.
		 */
		mask |= POLLIN;
		return mask;
	}

/* Wait on the rxq for an interrupt to wake us */
	poll_wait(filp, &shm->rxq, wait);

/* Check if the user's current RX entry is full */
	spin_lock_irqsave(&shm->rx_lock, flags);
	re = user_rx_entry(shm);
	if (re->own)
		/* XXX: We need a comment that explains what this does. */
		mask |= POLLIN | POLLRDNORM;

spin_unlock_irqrestore(&shm->rx_lock, flags);

return mask;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_chardev_ioctl --
 *
 *    Handle ioctls from user space.
 *
 * Results:
 *    Return code depends on ioctl.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static long
vmxnet3_shm_chardev_ioctl(struct file *filp,
		unsigned int cmd,
		unsigned long arg)
{
	struct vmxnet3_shm_pool *shm = filp->private_data;
	u16 idx;

switch(cmd) {

case SHM_IOCTL_TX:
			vmxnet3_shm_consume_user_tx_queue(shm);
			return 0;

case SHM_IOCTL_ALLOC_ONE:
			idx = vmxnet3_shm_alloc_page(shm);
			return idx;

case SHM_IOCTL_FREE_ONE:
			if (arg != SHM_INVALID_IDX && arg < shm->data.num_pages)
				vmxnet3_shm_free_page(shm, arg);

return 0;
	}

return -ENOTTY;
}

#ifndef HAVE_UNLOCKED_IOCTL
static int vmxnet3_shm_chardev_old_ioctl(struct inode *inode,
		struct file *filp,
		unsigned int cmd,
		unsigned long arg)
{
	return vmxnet3_shm_chardev_ioctl(filp, cmd, arg);
}
#endif

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_chardev_find_by_minor --
 *
 *    Find the right shared memory pool based on the minor number of the
 *    char device.
 *
 * Results:
 *    Pointer to the shared memory pool, or NULL on error.
 *
 * Side effects:
 *    Takes a reference on the kobj of the shm object.
 *
 *-----------------------------------------------------------------------------
 */

static struct vmxnet3_shm_pool *
vmxnet3_shm_chardev_find_by_minor(unsigned int minor)
{
	struct vmxnet3_shm_pool *shm, *tmp;
	unsigned long flags;

spin_lock_irqsave(&vmxnet3_shm_list_lock, flags);

list_for_each_entry_safe(shm, tmp, &vmxnet3_shm_list, list) {
		if (shm->misc_dev.minor == minor && kobject_get(&shm->kobj)) {
			spin_unlock_irqrestore(&vmxnet3_shm_list_lock, flags);
			return shm;
		}
	}

spin_unlock_irqrestore(&vmxnet3_shm_list_lock, flags);

return NULL;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_chardev_open --
 *
 *    Find the right shared memory pool based on the minor number of the
 *    char device.
 *
 * Results:
 *    0 on success or -ENODEV if no device exists with the given minor number
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static int
vmxnet3_shm_chardev_open(struct inode * inode,
		struct file * filp)
{
	/* Stash pointer to shm in file so file ops can use it */
	filp->private_data = vmxnet3_shm_chardev_find_by_minor(iminor(inode));
	if (filp->private_data == NULL)
		return -ENODEV;

/* XXX: What does this do?? */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
	/*   filp->f_mapping->backing_dev_info = &directly_mappable_cdev_bdi; */
#endif

return 0;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_chardev_release --
 *
 *    Closing the char device. Release the ref count on the shared memory
 *    pool, perform cleanup.
 *
 * Results:
 *    Always 0.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static int
vmxnet3_shm_chardev_release(struct inode * inode,
		struct file * filp)
{
	struct vmxnet3_shm_pool *shm = filp->private_data;

if (shm->adapter) {
		vmxnet3_shm_pool_reset(shm);
	} else {
		vmxnet3_shm_init_allocator(shm);
		memset(shm->ctl.ptr, 0, PAGE_SIZE);
	}

kobject_put(&shm->kobj);

return 0;
}

/* TX and RX */

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_free_skbpages --
 *
 *    Free the shared memory pages (secretly) backing this skb.
 *
 * Results:
 *    None.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

void
vmxnet3_free_skbpages(struct vmxnet3_adapter *adapter,
		struct sk_buff *skb)
{
	int i;

vmxnet3_shm_free_page(adapter->shm, VMXNET3_SHM_SKB_GETIDX(skb));
	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
		struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];

#if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
		vmxnet3_shm_free_page(adapter->shm, (unsigned long)frag->page);
#else
		vmxnet3_shm_free_page(adapter->shm, (unsigned long)frag->page.p);
#endif
	}

skb_shinfo(skb)->nr_frags = 0;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_start_tx --
 *
 *    The shared memory vmxnet version of the hard_start_xmit routine.
 *    Just frees the given packet as we do not intend to transmit any
 *    packet given to us by the TCP/IP stack.
 *
 * Results:
 *    Always 0 for success.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

int
vmxnet3_shm_start_tx(struct sk_buff *skb,
		struct net_device *dev)
{
	compat_dev_kfree_skb_irq(skb, FREE_WRITE);
	return COMPAT_NETDEV_TX_OK;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_tx_pkt --
 *
 *    Send a packet (collection of ring entries) using h/w tx routine.
 *
 *    Protected by shm.tx_lock
 *
 * Results:
 *    0 on success. Negative value to indicate error
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static int
vmxnet3_shm_tx_pkt(struct vmxnet3_adapter *adapter,
		struct vmxnet3_shm_ringentry *res,
		int frags)
{
	struct sk_buff* skb;
	int i;

skb = dev_alloc_skb(100);
	if (skb == NULL) {
		for (i = 0; i < frags; i++)
			vmxnet3_shm_free_page(adapter->shm, res[i].idx);

BUG_ON(TRUE);
		return -ENOMEM;
	}

VMXNET3_SHM_SKB_SETIDX(skb, res[0].idx);
	VMXNET3_SHM_SKB_SETLEN(skb, res[0].len);

for (i = 1; i < frags; i++) {
		struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
			skb_shinfo(skb)->nr_frags;

BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);

#if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
		frag->page = (struct page*)(unsigned long)res[i].idx;
#else
		__skb_frag_set_page(frag, (struct page*)(unsigned long)res[i].idx);
#endif

frag->page_offset = 0;
		frag->size = res[i].len;
		skb_shinfo(skb)->nr_frags ++;
	}

{
		/* shm always tx queue 0 */
		struct vmxnet3_tx_queue *tq = &adapter->tx_queue[0];
		int ret;
		skb->protocol = htons(ETH_P_IPV6);
		adapter->shm->ctl.ptr->stats.kernel_tx += frags; /* XXX: move to better place */
		ret = vmxnet3_shm_tq_xmit(skb, tq, adapter, adapter->netdev);
		if (ret == COMPAT_NETDEV_TX_BUSY) {
			vmxnet3_dev_kfree_skb(adapter, skb);
		}

return ret;
	}

return 0;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_tq_xmit --
 *
 *    Wrap vmxnet3_tq_xmit holding the netdev tx lock to better emulate the
 *    Linux stack. Also check for a stopped tx queue to avoid racing with
 *    vmxnet3_close.
 *
 * Results:
 *    Same as vmxnet3_tq_xmit.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */
int
vmxnet3_shm_tq_xmit(struct sk_buff *skb,
		struct vmxnet3_tx_queue *tq,
		struct vmxnet3_adapter *adapter,
		struct net_device *netdev)
{
	int ret = COMPAT_NETDEV_TX_BUSY;
	compat_netif_tx_lock(netdev);
	if (!netif_queue_stopped(netdev))
		ret = vmxnet3_tq_xmit(skb, tq, adapter, netdev);

compat_netif_tx_unlock(netdev);
	return ret;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_tx_re --
 *
 *    Add one entry to the partial TX array. If re->eop is set, i.e. if
 *    the packet is complete, TX the partial packet.
 *
 * Results:
 *    1 if eop
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static int
vmxnet3_shm_tx_re(struct vmxnet3_shm_pool *shm,
		struct vmxnet3_shm_ringentry re)
{
	int i;

BUG_ON(shm->partial_tx.frags > VMXNET3_SHM_MAX_FRAGS);
	if (shm->partial_tx.frags == VMXNET3_SHM_MAX_FRAGS) {
		if (re.eop) {
			printk("dropped oversize shm packet\n");
			for (i = 0; i < shm->partial_tx.frags; i++)
				vmxnet3_shm_free_page(shm,
						shm->partial_tx.res[i].idx);

shm->partial_tx.frags = 0;
		}
		vmxnet3_shm_free_page(shm, re.idx);
		return re.eop;
	}

shm->partial_tx.res[shm->partial_tx.frags++] = re;

if (re.eop) {
		int status = vmxnet3_shm_tx_pkt(shm->adapter,
				shm->partial_tx.res,
				shm->partial_tx.frags);
		if (status < 0)
			printk(KERN_DEBUG "vmxnet3_shm_tx_pkt failed %d\n", status);

shm->partial_tx.frags = 0;
		return 1;
	}

return 0;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_consume_user_tx_queue --
 *
 *    Consume all packets in the user TX queue and send full
 *    packets to the device
 *
 * Results:
 *    0 on success.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static int
vmxnet3_shm_consume_user_tx_queue(struct vmxnet3_shm_pool *shm)
{
	unsigned long flags;
	struct vmxnet3_shm_ringentry *re;

spin_lock_irqsave(&shm->tx_lock, flags);

/* Check if the device has been closed */
	if (shm->adapter == NULL) {
		spin_unlock_irqrestore(&shm->tx_lock, flags);
		return -1;
	}

/*
	 * Loop through each full entry in the user TX ring. Discard trash frags and
	 * add the others to the partial TX array. If an entry has eop set, TX the
	 * partial packet.
	 */
	while ((re = kernel_tx_entry(shm))->own) {
		if (re->trash) {
			vmxnet3_shm_free_page(shm, re->idx);
			shm->ctl.ptr->stats.kernel_tx++;
		} else {
			vmxnet3_shm_tx_re(shm, *re);
		}
		inc_kernel_tx_idx(shm);
		*re = RE_ZERO;
	}

spin_unlock_irqrestore(&shm->tx_lock, flags);

return 0;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_user_desc_available --
 *
 *    Checks if we have num_entries ring entries available on the rx ring.
 *
 * Results:
 *    0 for yes
 *    -ENOMEM for not enough entries available
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

static int
vmxnet3_shm_user_desc_available(struct vmxnet3_shm_pool *shm,
		u16 num_entries)
{
	struct vmxnet3_shm_ringentry *re;
	u16 reIdx = kernel_rx_idx(shm);

while (num_entries > 0) {
		re = &shm->ctl.ptr->rx_ring[reIdx];
		if (re->own)
			return -ENOMEM;

reIdx = (reIdx + 1) % SHM_RX_RING_SIZE;
		num_entries--;
	}

return 0;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_rx_skb --
 *
 *    Receives an skb into the rx ring. If we can't receive all fragments,
 *    the entire skb is dropped.
 *
 * Results:
 *    0 for success
 *    -ENOMEM for not enough entries available
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

int
vmxnet3_shm_rx_skb(struct vmxnet3_adapter *adapter,
		struct sk_buff *skb)
{
	int ret;
	int i;
	int num_entries = 1 + skb_shinfo(skb)->nr_frags;
	int eop = (num_entries == 1);

if (vmxnet3_shm_user_desc_available(adapter->shm, num_entries) == -ENOMEM) {
		vmxnet3_dev_kfree_skb_irq(adapter, skb);
		return -ENOMEM;
	}

ret = vmxnet3_shm_user_rx(adapter->shm,
			VMXNET3_SHM_SKB_GETIDX(skb),
			VMXNET3_SHM_SKB_GETLEN(skb),
			0 /* trash */,
			eop);
	if (ret != 0) {
		BUG_ON(TRUE);
		printk(KERN_ERR "vmxnet3_shm_user_rx failed on frag 0\n");
	}

for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
		struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
#if COMPAT_LINUX_VERSION_CHECK_LT(3, 2, 0)
		unsigned long shm_idx = (unsigned long)frag->page;
#else
		unsigned long shm_idx = (unsigned long)frag->page.p;
#endif

eop = (i == skb_shinfo(skb)->nr_frags - 1);

ret = vmxnet3_shm_user_rx(adapter->shm,
				shm_idx,
				frag->size,
				0 /* trash */,
				eop);
		if (ret != 0) {
			BUG_ON(TRUE);
			printk(KERN_ERR "vmxnet3_shm_user_rx failed on frag 1+\n");
		}
	}

/*
	 * Do NOT use the vmxnet3 version of kfree_skb, as we handed
	 * ownership of shm pages to the user space, thus we must not
	 * free them again.
	 */
	skb_shinfo(skb)->nr_frags = 0;
	compat_dev_kfree_skb_irq(skb, FREE_WRITE);

return 0;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_user_rx --
 *
 *    Put one packet fragment into the shared memory RX ring
 *
 * Results:
 *    0 on success.
 *    Negative value on error.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

int
vmxnet3_shm_user_rx(struct vmxnet3_shm_pool *shm,
		uint16 idx,
		uint16 len,
		int trash,
		int eop)
{
	struct vmxnet3_shm_ringentry *re = kernel_rx_entry(shm);
	shm->ctl.ptr->stats.kernel_rx++;
	if (re->own)
		return -ENOMEM;

inc_kernel_rx_idx(shm);
	re->idx = idx;
	re->len = len;
	re->trash = trash;
	re->eop = eop;
	re->own = TRUE;
	return 0;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_open --
 *
 *    Called when the vmxnet3 device is opened. Allocates the per-device
 *    shared memory pool.
 *
 * Results:
 *    0 on success.
 *    Negative value on error.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

int
vmxnet3_shm_open(struct vmxnet3_adapter *adapter,
		char *name, int pool_size)
{
	if (pool_size > SHM_MAX_DATA_SIZE) {
		printk(KERN_ERR "vmxnet3_shm: requested pool size %d is larger than the maximum %d\n",
		       pool_size, SHM_MAX_DATA_SIZE);
		return -EINVAL;
	}

adapter->shm = vmxnet3_shm_pool_create(adapter, name, pool_size);
	if (adapter->shm == NULL) {
		printk(KERN_ERR "failed to create shared memory pool\n");
		return -ENOMEM;
	}
	return 0;
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_shm_close --
 *
 *    Called when the vmxnet3 device is closed. Does not free the per-device
 *    shared memory pool. The character device might still be open. Thus
 *    freeing the shared memory pool is tied to the ref count on
 *    shm->kobj dropping to zero instead.
 *
 * Results:
 *    0 on success.
 *    Negative value on error.
 *
 * Side effects:
 *    None.
 *
 *-----------------------------------------------------------------------------
 */

int
vmxnet3_shm_close(struct vmxnet3_adapter *adapter)
{
	unsigned long flags;

/* Can't unset the lp pointer if a TX is in progress */
	spin_lock_irqsave(&adapter->shm->tx_lock, flags);
	adapter->shm->adapter = NULL;
	spin_unlock_irqrestore(&adapter->shm->tx_lock, flags);

/* Mark the channel as 'in bad state'  */
	adapter->shm->ctl.ptr->channelBad = 1;

// Wake up pollers so they see channelBad
	wake_up(&adapter->shm->rxq);

// Will not free shm pool if anyone is still using the char device
	kobject_put(&adapter->shm->kobj);

return 0;
}
�����������������������������������������������������������������������������������������������������������vmxnet3-only/README���������������������������������������������������������������������������������0000444�0000000�0000000�00000000736�13432725331�013026� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������The files in this directory are the source files for the VMware
3rd Generation Virtual Ethernet Adapter driver.  In order to build, 
make certain the Makefile is correct, especially about whether or 
not your system is multi-processor or not, and then just type:

make

from this directory.  A copy of the module will be left in 'vmxnet3.o',
which can then be installed in /lib/modules/<kernel-name>/net.

If you have any problems or questions, send mail to support@vmware.com
����������������������������������vmxnet3-only/vmxnet_def.h���������������������������������������������������������������������������0000444�0000000�0000000�00000013256�13432725350�014460� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 1999-2014 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

#ifndef _VMXNET_DEF_H_
#define _VMXNET_DEF_H_

#define INCLUDE_ALLOW_USERLEVEL

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMK_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#include "includeCheck.h"

#include "net_sg.h"
#include "vmnet_def.h"

/*
 *   Vmxnet I/O ports, used by both the vmxnet driver and 
 *   the device emulation code.
 */

#define VMXNET_INIT_ADDR		0x00
#define VMXNET_INIT_LENGTH		0x04
#define VMXNET_TX_ADDR		        0x08
#define VMXNET_COMMAND_ADDR		0x0c
#define VMXNET_MAC_ADDR			0x10
#define VMXNET_LOW_VERSION		0x18
#define VMXNET_HIGH_VERSION		0x1c
#define VMXNET_STATUS_ADDR		0x20
#define VMXNET_TOE_INIT_ADDR            0x24
#define VMXNET_APROM_ADDR               0x28
#define VMXNET_INT_ENABLE_ADDR          0x30
#define VMXNET_WAKE_PKT_PATTERNS        0x34

/*
 * Vmxnet command register values.
 */
#define VMXNET_CMD_INTR_ACK		0x0001
#define VMXNET_CMD_UPDATE_LADRF		0x0002
#define VMXNET_CMD_UPDATE_IFF		0x0004
#define VMXNET_CMD_UNUSED 1		0x0008
#define VMXNET_CMD_UNUSED_2		0x0010
#define VMXNET_CMD_INTR_DISABLE  	0x0020
#define VMXNET_CMD_INTR_ENABLE   	0x0040
#define VMXNET_CMD_UNUSED_3		0x0080
#define VMXNET_CMD_CHECK_TX_DONE	0x0100
#define VMXNET_CMD_GET_NUM_RX_BUFFERS	0x0200
#define VMXNET_CMD_GET_NUM_TX_BUFFERS	0x0400
#define VMXNET_CMD_PIN_TX_BUFFERS	0x0800
#define VMXNET_CMD_GET_CAPABILITIES	0x1000
#define VMXNET_CMD_GET_FEATURES		0x2000
#define VMXNET_CMD_SET_POWER_FULL       0x4000
#define VMXNET_CMD_SET_POWER_LOW        0x8000

/*
 * Vmxnet status register values.
 */
#define VMXNET_STATUS_CONNECTED		0x0001
#define VMXNET_STATUS_ENABLED		0x0002
#define VMXNET_STATUS_TX_PINNED         0x0004

/*
 * Values for the interface flags.
 */
#define VMXNET_IFF_PROMISC		0x01
#define VMXNET_IFF_BROADCAST		0x02
#define VMXNET_IFF_MULTICAST		0x04
#define VMXNET_IFF_DIRECTED             0x08

/*
 * Length of the multicast address filter.
 */
#define VMXNET_MAX_LADRF		2

/*
 * Size of Vmxnet APROM. 
 */
#define VMXNET_APROM_SIZE 6

/*
 * An invalid ring index.
 */
#define VMXNET_INVALID_RING_INDEX	(-1)

/*
 * Features that are implemented by the driver.  These are driver
 * specific so not all features will be listed here.  In addition not all
 * drivers have to pay attention to these feature flags.
 *
 *  VMXNET_FEATURE_ZERO_COPY_TX 	The driver won't do any copies as long as
 *					the packet length is > 
 *					Vmxnet_DriverData.minTxPhysLength.
 * 
 *  VMXNET_FEATURE_TSO                  The driver will use the TSO capabilities
 *                                      of the underlying hardware if available 
 *                                      and enabled.
 *
 *  VMXNET_FEATURE_JUMBO_FRAME          The driver can send/rcv jumbo frame 
 *
 *  VMXNET_FEATURE_LPD                  The backend can deliver large pkts
 */
#define VMXNET_FEATURE_ZERO_COPY_TX             0x01
#define VMXNET_FEATURE_TSO                      0x02
#define VMXNET_FEATURE_JUMBO_FRAME              0x04
#define VMXNET_FEATURE_LPD                      0x08

/*
 * Define the set of capabilities required by each feature above
 */
#define VMXNET_FEATURE_ZERO_COPY_TX_CAPS        VMXNET_CAP_SG
#define VMXNET_FEATURE_TSO_CAPS                 VMXNET_CAP_TSO
#define VMXNET_HIGHEST_FEATURE_BIT              VMXNET_FEATURE_TSO

#define VMXNET_INC(val, max)     \
   val++;                        \
   if (UNLIKELY(val == max)) {   \
      val = 0;                   \
   }

/*
 * code that just wants to switch on the different versions of the
 * guest<->implementation protocol can cast driver data to this.
 */
typedef uint32 Vmxnet_DDMagic;

/*
 * Wake packet pattern commands sent through VMXNET_WAKE_PKT_PATTERNS port
 */

#define VMXNET_PM_OPCODE_START 3 /* args: cnt of wake packet patterns */
#define VMXNET_PM_OPCODE_LEN   2 /* args: index of wake packet pattern */
                                 /*       number of pattern byte values */
#define VMXNET_PM_OPCODE_DATA  1 /* args: index of wake packet pattern */
                                 /*       offset in pattern byte values list */
                                 /*       packet byte offset */
                                 /*       packet byte value */
#define VMXNET_PM_OPCODE_END   0 /* args: <none> */

typedef union Vmxnet_WakePktCmd {
   uint32 pktData : 32;
   struct {
      unsigned cmd : 2; /* wake packet pattern cmd [from list above] */
      unsigned cnt : 3; /* cnt wk pkt pttrns 1..MAX_NUM_FILTER_PTTRNS */
      unsigned ind : 3; /* ind wk pkt pttrn 0..MAX_NUM_FILTER_PTTRNS-1 */
      unsigned lenOff : 8; /* num pttrn byte vals 1..MAX_PKT_FILTER_SIZE */
                           /* OR offset in pattern byte values list */
                           /* 0..MAX_PKT_FILTER_SIZE-1 */
      unsigned byteOff : 8; /* pkt byte offset 0..MAX_PKT_FILTER_SIZE-1 */
      unsigned byteVal : 8; /* packet byte value 0..255 */
   } pktPttrn;
} Vmxnet_WakePktCmd;

#endif /* _VMXNET_DEF_H_ */
��������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/eth_public.h���������������������������������������������������������������������������0000444�0000000�0000000�00000103143�13432725350�014432� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2005-2014,2017-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * eth.h  --
 *
 *    Virtual ethernet.
 */

#ifndef _ETH_PUBLIC_H_
#define _ETH_PUBLIC_H_

#define INCLUDE_ALLOW_MODULE
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMK_MODULE

#define INCLUDE_ALLOW_USERLEVEL
#include "includeCheck.h"

#include "vm_basic_defs.h"

#if defined __cplusplus
extern "C" {
#endif

#define ETH_LADRF_LEN      2
#define ETH_ADDR_LENGTH    6

typedef uint8 Eth_Address[ETH_ADDR_LENGTH];

// printf helpers
#define ETH_ADDR_FMT_STR     "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx"
#define ETH_ADDR_FMT_ARGS(a) ((uint8 *)a)[0], ((uint8 *)a)[1], ((uint8 *)a)[2], \
                             ((uint8 *)a)[3], ((uint8 *)a)[4], ((uint8 *)a)[5]
#define ETH_ADDR_PTR_FMT_ARGS(a) &((uint8 *)a)[0], &((uint8 *)a)[1], \
                                 &((uint8 *)a)[2], &((uint8 *)a)[3], \
                                 &((uint8 *)a)[4], &((uint8 *)a)[5]

#define ETH_MAX_EXACT_MULTICAST_ADDRS 32

typedef enum Eth_RxMode {
   ETH_FILTER_UNICAST   = 0x0001,   // pass unicast (directed) frames
   ETH_FILTER_MULTICAST = 0x0002,   // pass some multicast frames
   ETH_FILTER_ALLMULTI  = 0x0004,   // pass *all* multicast frames
   ETH_FILTER_BROADCAST = 0x0008,   // pass broadcast frames
   ETH_FILTER_PROMISC   = 0x0010,   // pass all frames (ie no filter)
   ETH_FILTER_USE_LADRF = 0x0020,   // use the LADRF for multicast filtering
   ETH_FILTER_SINK      = 0x10000   // pass not-matched unicast frames
} Eth_RxMode;

// filter flags printf helpers
#define ETH_FILTER_FLAG_FMT_STR     "%s%s%s%s%s%s%s"
#define ETH_FILTER_FLAG_FMT_ARGS(f) (f) & ETH_FILTER_UNICAST   ? "  UNICAST"   : "", \
                                    (f) & ETH_FILTER_MULTICAST ? "  MULTICAST" : "", \
                                    (f) & ETH_FILTER_ALLMULTI  ? "  ALLMULTI"  : "", \
                                    (f) & ETH_FILTER_BROADCAST ? "  BROADCAST" : "", \
                                    (f) & ETH_FILTER_PROMISC   ? "  PROMISC"   : "", \
                                    (f) & ETH_FILTER_USE_LADRF ? "  USE_LADRF" : "", \
                                    (f) & ETH_FILTER_SINK      ? "  SINK"      : ""

// Ethernet header type
typedef enum {
   ETH_HEADER_TYPE_DIX,
   ETH_HEADER_TYPE_802_1PQ,
   ETH_HEADER_TYPE_802_3,
   ETH_HEADER_TYPE_802_1PQ_802_3,
   ETH_HEADER_TYPE_NESTED_802_1PQ,
} Eth_HdrType;

// DIX type fields we care about
typedef enum {
   ETH_TYPE_IPV4        = 0x0800,
   ETH_TYPE_IPV6        = 0x86DD,
   ETH_TYPE_ARP         = 0x0806,
   ETH_TYPE_RARP        = 0x8035,
   ETH_TYPE_LLDP        = 0x88CC,
   ETH_TYPE_CDP         = 0x2000,
   ETH_TYPE_AKIMBI      = 0x88DE,
   ETH_TYPE_VMWARE      = 0x8922,
   ETH_TYPE_1588        = 0x88F7,
   ETH_TYPE_NSH         = 0x894F,
   ETH_TYPE_802_1PQ     = 0x8100,  // not really a DIX type, but used as such
   ETH_TYPE_QINQ        = 0x88A8,
   ETH_TYPE_LLC         = 0xFFFF,  // 0xFFFF is IANA reserved, used to mark LLC
} Eth_DixType;
typedef enum {
   ETH_TYPE_IPV4_NBO    = 0x0008,
   ETH_TYPE_IPV6_NBO    = 0xDD86,
   ETH_TYPE_ARP_NBO     = 0x0608,
   ETH_TYPE_RARP_NBO    = 0x3580,
   ETH_TYPE_LLDP_NBO    = 0xCC88,
   ETH_TYPE_CDP_NBO     = 0x0020,
   ETH_TYPE_AKIMBI_NBO  = 0xDE88,
   ETH_TYPE_VMWARE_NBO  = 0x2289,
   ETH_TYPE_1588_NBO    = 0xF788,
   ETH_TYPE_NSH_NBO     = 0x4F89,
   ETH_TYPE_802_1PQ_NBO = 0x0081,  // not really a DIX type, but used as such
   ETH_TYPE_QINQ_NBO    = 0xA888,
   ETH_TYPE_802_3_PAUSE_NBO = 0x0888,  // pause frame based ethernet flow control
} Eth_DixTypeNBO;

// low two bits of the LLC control byte
typedef enum {
   ETH_LLC_CONTROL_IFRAME  = 0x0, // both 0x0 and 0x2, only low bit of 0 needed
   ETH_LLC_CONTROL_SFRAME  = 0x1,
   ETH_LLC_CONTROL_UFRAME  = 0x3,
} Eth_LLCControlBits;

#define ETH_LLC_CONTROL_UFRAME_MASK (0x3)

typedef
#include "vmware_pack_begin.h"
struct Eth_DIX {
   uint16  typeNBO;     // indicates the higher level protocol
}
#include "vmware_pack_end.h"
Eth_DIX;

/*
 * LLC header come in two varieties:  8 bit control and 16 bit control.
 * when the lower two bits of the first byte's control are '11', this
 * indicated the 8 bit control field.
 */

typedef 
#include "vmware_pack_begin.h"
struct Eth_LLC8 {
   uint8   dsap;
   uint8   ssap;
   uint8   control;
}
#include "vmware_pack_end.h"
Eth_LLC8;

typedef
#include "vmware_pack_begin.h"
struct Eth_LLC16 {
   uint8   dsap;
   uint8   ssap;
   uint16  control;
}
#include "vmware_pack_end.h"
Eth_LLC16;

typedef
#include "vmware_pack_begin.h"
struct Eth_SNAP {
   uint8   snapOrg[3];
   Eth_DIX snapType;
} 
#include "vmware_pack_end.h"
Eth_SNAP;

typedef
#include "vmware_pack_begin.h"
struct Eth_802_3 {  
   uint16   lenNBO;      // length of the frame
   Eth_LLC8 llc;         // LLC header
   Eth_SNAP snap;        // SNAP header
} 
#include "vmware_pack_end.h"
Eth_802_3;

// 802.1p QOS/priority tags
// 
enum {
   ETH_802_1_P_BEST_EFFORT          = 0,
   ETH_802_1_P_BACKGROUND           = 1,
   ETH_802_1_P_EXCELLENT_EFFORT     = 2,
   ETH_802_1_P_CRITICAL_APPS        = 3,
   ETH_802_1_P_VIDEO                = 4,
   ETH_802_1_P_VOICE                = 5,
   ETH_802_1_P_INTERNETWORK_CONROL  = 6,
   ETH_802_1_P_NETWORK_CONTROL      = 7
};

typedef
#include "vmware_pack_begin.h"
struct Eth_802_1pq_Tag {
   uint16 typeNBO;            // always ETH_TYPE_802_1PQ
   uint16 vidHi:4,            // 802.1q vlan ID high nibble
          canonical:1,        // bit order? (should always be 0)
          priority:3,         // 802.1p priority tag
          vidLo:8;            // 802.1q vlan ID low byte
}
#include "vmware_pack_end.h"
Eth_802_1pq_Tag;

typedef
#include "vmware_pack_begin.h"
struct Eth_802_1pq {
   Eth_802_1pq_Tag tag;       // VLAN/QOS tag
   union {
      Eth_DIX      dix;       // DIX header follows
      Eth_802_3    e802_3;    // or 802.3 header follows 
      struct {
         Eth_802_1pq_Tag   tag;        // inner VLAN/QOS tag
         Eth_DIX           dix;        // DIX header follows
      } nested802_1pq;
   }; 
}
#include "vmware_pack_end.h"
Eth_802_1pq;

typedef
#include "vmware_pack_begin.h"
struct Eth_Header {
   Eth_Address     dst;       // all types of ethernet frame have dst first
   Eth_Address     src;       // and the src next (at least all the ones we'll see)
   union {
      Eth_DIX      dix;       // followed by a DIX header...
      Eth_802_3    e802_3;    // ...or an 802.3 header
      Eth_802_1pq  e802_1pq;  // ...or an 802.1[pq] tag and a header
   };
}
#include "vmware_pack_end.h"
Eth_Header;

/*
 * Official VMware ethertype header format and types
 */
#define ETH_VMWARE_FRAME_MAGIC   0x026f7564

enum {
   ETH_VMWARE_FRAME_TYPE_INVALID    = 0,
   ETH_VMWARE_FRAME_TYPE_BEACON     = 1,
   ETH_VMWARE_FRAME_TYPE_COLOR      = 2,
   ETH_VMWARE_FRAME_TYPE_ECHO       = 3,
   ETH_VMWARE_FRAME_TYPE_LLC        = 4, // XXX: Just re-use COLOR?
};

typedef
#include "vmware_pack_begin.h"
struct Eth_VMWareFrameHeader {
   uint32         magic;
   uint16         lenNBO;
   uint8          type;
}
#include "vmware_pack_end.h"
Eth_VMWareFrameHeader;

typedef Eth_Header Eth_802_1pq_Header; // for sizeof

#define ETH_BROADCAST_ADDRESS { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }

extern Eth_Address netEthBroadcastAddr;

/*
 * simple predicate for 1536 boundary.
 * the parameter is a network ordered uint16, which is compared to 0x06,
 * testing for "length" values greater than or equal to 0x0600 (1536)
 */

#define ETH_TYPENOT8023(x)      (((x) & 0xff) >= 0x06)

/*
 * header length macros
 *
 * first two are typical: ETH_HEADER_LEN_DIX, ETH_HEADER_LEN_802_1PQ
 * last two are suspicious, due to 802.3 incompleteness
 */

#define ETH_HEADER_LEN_DIX           (sizeof(Eth_Address) + \
                                      sizeof(Eth_Address) + \
                                      sizeof(Eth_DIX))
#define ETH_HEADER_LEN_802_1PQ       (sizeof(Eth_Address) + \
                                      sizeof(Eth_Address) + \
                                      sizeof(Eth_802_1pq_Tag) + \
                                      sizeof(Eth_DIX))
#define ETH_HEADER_LEN_802_2_LLC     (sizeof(Eth_Address) + \
                                      sizeof(Eth_Address) + \
                                      sizeof(uint16) + \
                                      sizeof(Eth_LLC8))
#define ETH_HEADER_LEN_802_2_LLC16   (sizeof(Eth_Address) + \
                                      sizeof(Eth_Address) + \
                                      sizeof(uint16) + \
                                      sizeof(Eth_LLC16))
#define ETH_HEADER_LEN_802_3         (sizeof(Eth_Address) + \
                                      sizeof(Eth_Address) + \
                                      sizeof(Eth_802_3))
#define ETH_HEADER_LEN_802_1PQ_LLC   (sizeof(Eth_Address) + \
                                      sizeof(Eth_Address) + \
                                      sizeof(Eth_802_1pq_Tag) + \
                                      sizeof(uint16) + \
                                      sizeof(Eth_LLC8))
#define ETH_HEADER_LEN_802_1PQ_LLC16 (sizeof(Eth_Address) + \
                                      sizeof(Eth_Address) + \
                                      sizeof(Eth_802_1pq_Tag) + \
                                      sizeof(uint16) + \
                                      sizeof(Eth_LLC16))
#define ETH_HEADER_LEN_802_1PQ_802_3 (sizeof(Eth_Address) + \
                                      sizeof(Eth_Address) + \
                                      sizeof(Eth_802_1pq_Tag) + \
                                      sizeof(Eth_802_3))
#define ETH_HEADER_LEN_NESTED_802_1PQ  (sizeof(Eth_Address) + \
                                        sizeof(Eth_Address) + \
                                        sizeof(Eth_802_1pq_Tag) + \
                                        sizeof(Eth_802_1pq_Tag) + \
                                        sizeof(Eth_DIX))

#define ETH_MIN_HEADER_LEN   (ETH_HEADER_LEN_DIX)
#define ETH_MAX_HEADER_LEN   (ETH_HEADER_LEN_802_1PQ_802_3)

#define ETH_MIN_FRAME_LEN                    60
#define ETH_MAX_STD_MTU                      1500
#define ETH_MAX_STD_FRAMELEN                 (ETH_MAX_STD_MTU + ETH_MAX_HEADER_LEN)
#define ETH_MAX_JUMBO_MTU                    9000
#define ETH_MAX_JUMBO_FRAMELEN               (ETH_MAX_JUMBO_MTU + ETH_MAX_HEADER_LEN)

#define ETH_DEFAULT_MTU                      1500

#define ETH_FCS_LEN                          4
#define ETH_VLAN_LEN                         sizeof(Eth_802_1pq_Tag)

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsAddrMatch --
 *
 *      Do the two ethernet addresses match?
 *
 * Results: 
 *	TRUE or FALSE.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsAddrMatch(const Eth_Address addr1, const Eth_Address addr2)
{
#ifdef __GNUC__
   /* string.h may not exist for kernel modules, so cannot use memcmp() */
   return !__builtin_memcmp(addr1, addr2, ETH_ADDR_LENGTH);
#else
   return !memcmp(addr1, addr2, ETH_ADDR_LENGTH);
#endif
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsBroadcastAddr --
 *
 *      Is the address the broadcast address?
 *
 * Results: 
 *	TRUE or FALSE.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsBroadcastAddr(const Eth_Address addr) 
{
   return Eth_IsAddrMatch(addr, netEthBroadcastAddr);
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsUnicastAddr --
 *
 *      Is the address a unicast address?
 *
 * Results: 
 *	TRUE or FALSE.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsUnicastAddr(const Eth_Address addr) 
{
   // broadcast and multicast frames always have the low bit set in byte 0 
   return !(((char *)addr)[0] & 0x1);
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsNullAddr --
 *
 *      Is the address the all-zeros address?
 *
 * Results: 
 *	TRUE or FALSE.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsNullAddr(const Eth_Address addr) 
{
   return ((addr[0] | addr[1] | addr[2] | addr[3] | addr[4] | addr[5]) == 0);
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_HeaderType --
 *
 *      return an Eth_HdrType depending on the eth header
 *      contents.  will not work in all cases, especially since it
 *      requres ETH_HEADER_LEN_802_1PQ bytes to determine the type
 *
 *      HeaderType isn't sufficient to determine the length of
 *      the eth header.  for 802.3 header, its not clear without
 *      examination, whether a SNAP is included
 *
 *      returned type:
 *
 *      ETH_HEADER_TYPE_DIX: typical 14 byte eth header
 *      ETH_HEADER_TYPE_802_1PQ: DIX+vlan tagging
 *      ETH_HEADER_TYPE_NESTED_802_1PQ: DIX+vlan tagging+DIX+vlan tagging
 *      ETH_HEADER_TYPE_802_3: 802.3 eth header
 *      ETH_HEADER_TYPE_802_1PQ_802_3: 802.3 + vlan tag 
 *
 *      the test for DIX was moved from a 1500 boundary to a 1536
 *      boundary, since the vmxnet2 MTU was updated to 1514.  when
 *      W2K8 attempted to send LLC frames, these were interpreted
 *      as DIX frames instead of the correct 802.3 type
 *
 *      these links may help if they're valid:
 *
 *      http://standards.ieee.org/regauth/ethertype/type-tut.html
 *      http://standards.ieee.org/regauth/ethertype/type-pub.html
 *      
 *
 * Results: 
 *	Eth_HdrType value
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Eth_HdrType
Eth_HeaderType(const Eth_Header *eh)
{
   /*
    * we use 1536 (IEEE 802.3-std mentions 1536, but iana indicates
    * type of 0-0x5dc are 802.3) instead of some #def symbol to prevent
    * inadvertant reuse of the same macro for buffer size decls.
    */

if (ETH_TYPENOT8023(eh->dix.typeNBO)) {
      if (eh->dix.typeNBO != ETH_TYPE_802_1PQ_NBO) {

/*
          * typical case
          */

return ETH_HEADER_TYPE_DIX;
      }

/*
       * some type of 802.1pq tagged frame
       */

if (ETH_TYPENOT8023(eh->e802_1pq.dix.typeNBO)) {

/*
          * vlan tagging with dix style type
          */

if (UNLIKELY(eh->e802_1pq.dix.typeNBO == ETH_TYPE_802_1PQ_NBO)) {
            return ETH_HEADER_TYPE_NESTED_802_1PQ;
         }

return ETH_HEADER_TYPE_802_1PQ;
      }

/*
       * vlan tagging with 802.3 header
       */

return ETH_HEADER_TYPE_802_1PQ_802_3;
   }
   
   /*
    * assume 802.3
    */

return ETH_HEADER_TYPE_802_3;
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_EncapsulatedPktType --
 *
 *      Get the encapsulated (layer 3) frame type. 
 *      for LLC frames without SNAP, we don't have
 *      an encapsulated type, and return ETH_TYPE_LLC.
 *
 *      IANA reserves 0xFFFF, which we reuse to indicate
 *      ETH_TYPE_LLC.  
 *      
 *
 * Results: 
 *	NBO frame type.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE uint16
Eth_EncapsulatedPktType(const Eth_Header *eh)
{
   Eth_HdrType type = Eth_HeaderType(eh);

switch (type) {
   case ETH_HEADER_TYPE_DIX :
      return eh->dix.typeNBO;

case ETH_HEADER_TYPE_802_1PQ :
      return eh->e802_1pq.dix.typeNBO;

case ETH_HEADER_TYPE_NESTED_802_1PQ:
      return eh->e802_1pq.nested802_1pq.dix.typeNBO;

case ETH_HEADER_TYPE_802_3 :
      /*
       * Documentation describes SNAP headers as having ONLY
       * 0x03 as the control fields, not just the lower two bits
       * This prevents the use of Eth_IsLLCControlUFormat.
       */
      if ((eh->e802_3.llc.dsap == 0xaa) &&
           (eh->e802_3.llc.ssap == 0xaa) &&
           (eh->e802_3.llc.control == ETH_LLC_CONTROL_UFRAME)) {
               return eh->e802_3.snap.snapType.typeNBO;
      } else {
         // LLC, no snap header, then no type
         return ETH_TYPE_LLC;
      }

case ETH_HEADER_TYPE_802_1PQ_802_3 :
      if ((eh->e802_1pq.e802_3.llc.dsap == 0xaa) &&
           (eh->e802_1pq.e802_3.llc.ssap == 0xaa) &&
           (eh->e802_1pq.e802_3.llc.control == ETH_LLC_CONTROL_UFRAME)) {
               return eh->e802_1pq.e802_3.snap.snapType.typeNBO;
      } else {
         // tagged LLC, no snap header, then no type
         return ETH_TYPE_LLC;
      }
   }

ASSERT(FALSE);
   return 0;
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsDixType --
 *
 *      Is the frame of the requested protocol type or is it an 802.1[pq]
 *      encapsulation of such a frame?
 *
 * Results: 
 *	TRUE or FALSE.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsDixType(const Eth_Header *eh, const Eth_DixTypeNBO type) 
{
   return Eth_EncapsulatedPktType(eh) == type;
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsBeaconSap --
 *
 *      test to validate a frame to determine if its a beacon
 *      (ncp) frame.  sap is passed in.
 *
 *      ncp beacon/color frames are LLC frames with a DSAP/SSAP
 *      set based on a config value.  non-zero llc length is 
 *      tested here to prevent the predicate from interfering
 *      with testworld etherswitch tests
 *
 *
 * Results:
 *      TRUE or FALSE.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsBeaconSap(const Eth_Header *eh, const uint8 sap)
{
   Eth_HdrType type = Eth_HeaderType(eh);

if (type == ETH_HEADER_TYPE_802_3) {
      if ((eh->e802_3.llc.dsap == sap) && (eh->e802_3.llc.ssap == sap)) {
         if (eh->e802_3.lenNBO != 0) {
            return TRUE;
         }
      }
   } else if (type == ETH_HEADER_TYPE_802_1PQ_802_3) {
      if ((eh->e802_1pq.e802_3.llc.dsap == sap) &&
          (eh->e802_1pq.e802_3.llc.ssap == sap)) {
            if (eh->e802_1pq.e802_3.lenNBO != 0) {
               return TRUE;
            }
      }
   }
   return FALSE;
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsIPV4 --
 *
 *      Is the frame an IPV4 frame?
 *
 * Results: 
 *	TRUE or FALSE.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsIPV4(const Eth_Header *eh) 
{
   return Eth_IsDixType(eh, ETH_TYPE_IPV4_NBO);
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsIPV6 --
 *
 *      Is the frame an IPV6 frame?
 *
 * Results: 
 *	TRUE or FALSE.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsIPV6(const Eth_Header *eh) 
{
   return Eth_IsDixType(eh, ETH_TYPE_IPV6_NBO);
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsVMWare --
 *
 *      Is the frame a VMWare frame?
 *
 * Results: 
 *	TRUE or FALSE.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsVMWare(const Eth_Header *eh) 
{
   return Eth_IsDixType(eh, ETH_TYPE_VMWARE_NBO);
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsARP --
 *
 *      Is the frame an ARP frame?
 *
 * Results: 
 *	TRUE or FALSE.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsARP(const Eth_Header *eh) 
{
   return Eth_IsDixType(eh, ETH_TYPE_ARP_NBO);
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsFrameTagged --
 *
 *      Does the frame contain an 802.1[pq] tag?
 *
 * Results: 
 *	TRUE or FALSE.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsFrameTagged(const Eth_Header *eh) 
{
   return (eh->dix.typeNBO == ETH_TYPE_802_1PQ_NBO);
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsPauseFrame --
 *
 *      Is the frame 802.3 pause frame ?
 *
 * Results:
 *	TRUE or FALSE.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsPauseFrame(const Eth_Header *eh)
{
   return (eh->dix.typeNBO == ETH_TYPE_802_3_PAUSE_NBO);
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_FillVlanTag --
 *
 *      Populate the fields of a vlan tag
 *
 * Results: 
 *	The populated vlan tag
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Eth_802_1pq_Tag *
Eth_FillVlanTag(Eth_802_1pq_Tag *tag,
                const uint32 vlanId,
                const uint32 priority)
{

ASSERT(vlanId < 4096);
   ASSERT(priority < 8);

tag->typeNBO = ETH_TYPE_802_1PQ_NBO;
   tag->priority = priority;
   tag->canonical = 0;                  // bit order (should be 0)
   tag->vidHi = vlanId >> 8;
   tag->vidLo = vlanId & 0xff;

return tag;
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_VlanTagGetVlanID --
 *
 *      Extract the VLAN ID from the vlanTag.
 *
 * Results: 
 *	The VLAN ID.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE uint16
Eth_VlanTagGetVlanID(const Eth_802_1pq_Tag *tag)
{
   return (tag->vidHi << 8) | tag->vidLo;
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_FrameGetVlanID --
 *
 *      Extract the VLAN ID from the frame's 802.1[pq] tag.
 *
 * Results: 
 *	The VLAN ID.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE uint16
Eth_FrameGetVlanID(const Eth_Header *eh) 
{
   ASSERT(Eth_IsFrameTagged(eh));

return Eth_VlanTagGetVlanID(&eh->e802_1pq.tag);
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_FrameSetVlanID --
 *
 *      Set the VLAN ID in the frame's 802.1[pq] tag.
 *
 * Results: 
 *	None.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE void
Eth_FrameSetVlanID(Eth_Header *eh, uint16 vid) 
{
   ASSERT(Eth_IsFrameTagged(eh));

eh->e802_1pq.tag.vidHi = vid >> 8;
   eh->e802_1pq.tag.vidLo = vid & 0xff;
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_FrameGetPriority --
 *
 *      Extract the priority from the frame's 802.1[pq] tag.
 *
 * Results: 
 *	The priority.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE uint8
Eth_FrameGetPriority(const Eth_Header *eh) 
{
   ASSERT(Eth_IsFrameTagged(eh));

return eh->e802_1pq.tag.priority;
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_FrameSetPriority --
 *
 *      Set the priority in the frame's 802.1[pq] tag.
 *
 * Results: 
 *	None.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static INLINE void
Eth_FrameSetPriority(Eth_Header *eh, const uint8 prio) 
{
   ASSERT(Eth_IsFrameTagged(eh));
   ASSERT(prio <= 7);

eh->e802_1pq.tag.priority = prio;
}

/*
 *----------------------------------------------------------------------
 * Eth_IsLLCControlUFormat --
 *
 *      The LLC Control fields determines the lengeth of the LLC
 *      field, selecting 8 bit of 16 bit.  Thies predicate indicates
 *      whether the LLC frame is a U-Format frames.  The U-Format
 *      frame is the only LLC frame header which is 8 bits long.  
 *
 * Results
 *      Bool
 *      
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsLLCControlUFormat(const uint8 control)
{
   return (control & ETH_LLC_CONTROL_UFRAME_MASK) == ETH_LLC_CONTROL_UFRAME;
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_HeaderLength_802_3 --
 *
 *      Returns the length of an 802_3 eth header without
 *      any vlan tagging.  factored out for Eth_HeaderComplete()
 *
 * Results:
 *      uint16 length
 *
 * Side Effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */
static INLINE uint16
Eth_HeaderLength_802_3(const Eth_Header *eh)
{
   /*
    * Documentation describes SNAP headers as having ONLY
    * 0x03 as the control fields, not just the lower two bits
    * This prevents the use of Eth_IsLLCControlUFormat.
    */
   if ((eh->e802_3.llc.dsap == 0xaa) &&
        (eh->e802_3.llc.ssap == 0xaa) &&
        (eh->e802_3.llc.control == ETH_LLC_CONTROL_UFRAME)) {
            return ETH_HEADER_LEN_802_3;
   }
   // LLC, no snap header
   if (Eth_IsLLCControlUFormat(eh->e802_3.llc.control)) {
      return ETH_HEADER_LEN_802_2_LLC;
   }
   // Eth_LLC with a two byte control field
   return ETH_HEADER_LEN_802_2_LLC16;
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_HeaderLength_802_1PQ_802_3 --
 *
 *      Returns the length of an 802_3 eth header with 
 *      vlan tagging.  factored out for Eth_HeaderComplete()
 *
 * Results:
 *      uint16 length
 *
 * Side Effects:
 *      None.
 *
 *
 *----------------------------------------------------------------------
 */
static INLINE uint16
Eth_HeaderLength_802_1PQ_802_3(const Eth_Header *eh)
{
   if ((eh->e802_1pq.e802_3.llc.dsap == 0xaa) &&
        (eh->e802_1pq.e802_3.llc.ssap == 0xaa) &&
        (eh->e802_1pq.e802_3.llc.control == ETH_LLC_CONTROL_UFRAME)) {
            return ETH_HEADER_LEN_802_1PQ_802_3;
   }
   // tagged LLC, no snap header
   if (Eth_IsLLCControlUFormat(eh->e802_1pq.e802_3.llc.control)) {
      return ETH_HEADER_LEN_802_1PQ_LLC;
   }
   // Eth_LLC with a two byte control field
   return ETH_HEADER_LEN_802_1PQ_LLC16;
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_HeaderLength --
 *
 *      Return the length of the header, taking into account
 *      different header variations.  for LLC headers,  determine
 *      whether the eth header has an associated SNAP header,
 *
 *      all references to the eth header assume the complete
 *      frame is mapped withing the frame mapped length
 *
 *      To determine the length, 17 bytes must be readable.
 *      LLC requires three bytes (after the 802.3 length) to 
 *      identify SNAP frames.
 *
 *      when the header isn't complete:
 *      Eth_HeaderType needs ETH_HEADER_LEN_DIX + sizeof(Eth_802_1pq_Tag)
 *      to completely distinguish types (18 bytes), 
 *      Eth_HeaderType correctly identifies basic untagged DIX frames
 *      with ETH_HEADER_LEN_DIX (14 bytes) bytes.
 *      Eth_HeaderLength will correctly return length of untagged LLC frames 
 *      with ETH_HEADER_LEN_DIX + sizeof(Eth_LLC) (17 bytes), but if the
 *      frame is tagged, it will need
 *      ETH_HEADER_LEN_DIX + sizeof(Eth_802_1pq_Tag) + sizeof(Eth_LLC)
 *      (21 bytes)
 *
 *
 * Results:
 *      uint16 size of the header
 *
 * Side Effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE uint16
Eth_HeaderLength(const Eth_Header *eh)
{
   Eth_HdrType type = Eth_HeaderType(eh);

switch (type) {
   case ETH_HEADER_TYPE_DIX :
      return ETH_HEADER_LEN_DIX;

case ETH_HEADER_TYPE_802_1PQ :
      return ETH_HEADER_LEN_802_1PQ;

case ETH_HEADER_TYPE_NESTED_802_1PQ :
      return ETH_HEADER_LEN_NESTED_802_1PQ;

case ETH_HEADER_TYPE_802_3 :
      return Eth_HeaderLength_802_3(eh);

case ETH_HEADER_TYPE_802_1PQ_802_3 :
      return Eth_HeaderLength_802_1PQ_802_3(eh);
   }
   
   ASSERT(FALSE);
   return 0;   
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_GetPayloadWithLen --
 *
 *      Simple cover to comput the payload's address given the length
 *
 * Results:
 *      pointer to the frame's payload
 *
 * Side Effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE void *
Eth_GetPayloadWithLen(const void *frame, const uint16 ehHdrLen)
{
   return (void *)((uint8 *)frame + ehHdrLen);
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_GetPayload --
 *
 *      Return the address of the frame's payload, taking into account
 *      different header variations.  Code assumes a complete ether
 *      header is mapped at the frame, assumption exists due to
 *      the use of Eth_HeaderLength, which isn't provided with
 *      the frame's length (how much data it can examine to determine
 *      the ethernet header length)
 *
 * Results:
 *      pointer to the frame's payload
 *
 * Side Effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE void *
Eth_GetPayload(const void *frame)
{
   return Eth_GetPayloadWithLen(frame, Eth_HeaderLength((Eth_Header *)frame));
}

/*
 *----------------------------------------------------------------------
 *
 * Eth_IsFrameHeaderComplete -- 
 *
 *      Predicate to determine whether the frame has enough length
 *      to contain a complete eth header of the correct type.
 *      If you already know/expect the length to exceed
 *      ETH_MAX_HEADER_LEN then for performance reasons you should
 *      explicitly check for that before calling this function.
 *
 * Results:
 *      returns true when a complete eth header is available
 *
 * Side Effects:
 *      None.
 *
 *----------------------------------------------------------------------
 */

static INLINE Bool
Eth_IsFrameHeaderComplete(const Eth_Header *eh,
                          const uint32 len,
                          uint16 *ehHdrLen)
{
   uint16 ehLen;

if (UNLIKELY(eh == NULL)) {
      return FALSE;
   }
   /*
    * Hard to know what's the optimal order for these checks
    * perform the most likely case first.
    * Since its not directly obvious, hex 0x06 -> 1536, 
    * (see ETH_TYPENOT8023() for 1536 details)
    */
   if (LIKELY((len >= ETH_HEADER_LEN_DIX) &&
              ETH_TYPENOT8023(eh->dix.typeNBO) &&
              (eh->dix.typeNBO != ETH_TYPE_802_1PQ_NBO))) {
   
      if (ehHdrLen != NULL) {
         *ehHdrLen = ETH_HEADER_LEN_DIX;
      }
      return TRUE;
   }
   if (len >= ETH_HEADER_LEN_802_1PQ) {
      /*
       * Eth_HeaderType will correctly enumerate all types once
       * at least ETH_HEADER_LEN_802_1PQ bytes are available except nested
       * 802.1pq tag.
       */
      Eth_HdrType type = Eth_HeaderType(eh);

switch (type) {
      case ETH_HEADER_TYPE_802_1PQ:
         if (ehHdrLen != NULL) {
            *ehHdrLen = ETH_HEADER_LEN_802_1PQ;
         }
         return TRUE;

case ETH_HEADER_TYPE_NESTED_802_1PQ:
         if (ehHdrLen != NULL) {
            *ehHdrLen = ETH_HEADER_LEN_NESTED_802_1PQ;
         }
         return len >= ETH_HEADER_LEN_NESTED_802_1PQ;

case ETH_HEADER_TYPE_802_3:
         /*
          * Length could be shorter LLC or LLC+SNAP.
          * ETH_HEADER_LEN_802_2_LLC bytes are needed to disambiguate.
          * note: ASSERT_ON_COMPILE fails windows builds.
          */
         ASSERT(ETH_HEADER_LEN_802_1PQ > ETH_HEADER_LEN_802_2_LLC);
         ehLen = Eth_HeaderLength_802_3(eh);
         /* continue to common test */
         break;

case ETH_HEADER_TYPE_802_1PQ_802_3:
         if (len < ETH_HEADER_LEN_802_1PQ_LLC) {
            return FALSE;
         }
         ehLen = Eth_HeaderLength_802_1PQ_802_3(eh);
         /* continue to common test */
         break;

default:
         /*
          * This else clause is unreachable, but if removed
          * compiler complains about ehLen possibly being
          * uninitialized.  this else is marginally preferred
          * over an unnecessary initialization
          */
         ASSERT(type != ETH_HEADER_TYPE_DIX);
         // NOT_REACHED()
         return FALSE;
      }

/* common test */
      if (len >= ehLen) {
         if (ehHdrLen != NULL) {
            *ehHdrLen = ehLen;
         }
         return TRUE;
      }
      return FALSE;
   }

/*
    * Corner case...  not enough len to use Eth_HeaderType,
    * since len is shorter than ETH_HEADER_LEN_802_1PQ bytes. 
    * but with ETH_HEADER_LEN_802_2_LLC bytes, and an 802.3
    * frame, a U Format LLC frame indicates TRUE 
    * with a header length of ETH_HEADER_LEN_802_2_LLC bytes.
    *
    * The additional test for ETH_TYPENOT8023() is necessary
    * for the case where the dix frame failed due to the
    * vlan tagging test early in this procedure.
    */
   if ((len == ETH_HEADER_LEN_802_2_LLC) &&
       !ETH_TYPENOT8023(eh->dix.typeNBO) &&
       Eth_IsLLCControlUFormat(eh->e802_3.llc.control)) {

if (ehHdrLen != NULL) {
         *ehHdrLen = ETH_HEADER_LEN_802_2_LLC;
      }
      return TRUE;
   }
   return FALSE;
}

#if defined __cplusplus
} // extern "C"
#endif

#endif // _ETH_PUBLIC_H_
�����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/vmxnet3_shm.h��������������������������������������������������������������������������0000444�0000000�0000000�00000016662�13432725331�014577� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2009 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmxnet3_shm.h --
 *
 *      Definitions for shared memory infrastructure for VMXNET3 linux driver.
 */

#ifndef _VMXNET3_SHM_H_
#define _VMXNET3_SHM_H_

#include "vmxnet3_shm_shared.h"
#include <linux/miscdevice.h>

/*
 * Bumping up the max tx descriptor per packet.
 * We need one more than VMXNET3_SHM_MAX_FRAGS because of partial header copy.
 */
#define VMXNET3_MAX_TXD_PER_PKT_SHM (VMXNET3_SHM_MAX_FRAGS + 1)

struct vmxnet3_shm_mapped_page
{
	struct page *page;
	void *virt;
};

struct vmxnet3_shm_pool
{
	struct list_head list;
	char name[IFNAMSIZ + 16];
	struct kobject kobj;

struct
	{
		/* pages backing the map in virtual address order */
		struct vmxnet3_shm_mapped_page *pages;
		unsigned int num_pages;
	} data;

struct
	{
		/* pages backing the map in virtual address order */
		struct page *pages[SHM_CTL_SIZE];
		struct vmxnet3_shm_ctl *ptr;
	} ctl;

struct
	{
		/*
		 * This is a stack of free pages. count is the number of free pages, so
		 * count - 1 is the topmost free page.
		 */
		u16 count;
		u16 *stack;
	} allocator;

struct
	{
		struct vmxnet3_shm_ringentry res[VMXNET3_SHM_MAX_FRAGS];
		int frags;
	} partial_tx;

struct miscdevice misc_dev;

wait_queue_head_t rxq;
	spinlock_t alloc_lock, tx_lock, rx_lock;
	struct vmxnet3_adapter *adapter;
};

/* Convert ring index to the struct page* or virt address. */
#define VMXNET3_SHM_IDX2PAGE(shm, idx) (shm->data.pages[(idx)].page)
#define VMXNET3_SHM_SET_IDX2PAGE(shm, idx, x) (shm->data.pages[(idx)].page = (x))

#define VMXNET3_SHM_SKB_GETIDX(skb) (compat_skb_transport_offset(skb))
#define VMXNET3_SHM_SKB_SETIDX(skb, idx) (compat_skb_set_transport_header(skb, idx))
#define VMXNET3_SHM_SKB_SETLEN(skb, len) (compat_skb_set_network_header(skb, len))
#define VMXNET3_SHM_SKB_GETLEN(skb) (compat_skb_network_offset(skb))

int
vmxnet3_shm_close(struct vmxnet3_adapter *adapter);
int
vmxnet3_shm_open(struct vmxnet3_adapter *adapter, char *name, int pool_size);
int
vmxnet3_shm_user_rx(struct vmxnet3_shm_pool *shm,
		u16 idx, u16 len,
		int trash, int eop);
void
vmxnet3_free_skbpages(struct vmxnet3_adapter *adapter, struct sk_buff *skb);

u16
vmxnet3_shm_alloc_page(struct vmxnet3_shm_pool *shm);
void
vmxnet3_shm_free_page(struct vmxnet3_shm_pool *shm, u16 idx);

int
vmxnet3_shm_start_tx(struct sk_buff *skb, struct net_device *dev);
int
vmxnet3_shm_rx_skb(struct vmxnet3_adapter *adapter, struct sk_buff *skb);

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_dev_kfree_skb_* --
 *
 *      Covers for dev_kfree_skb*. Deal with the shared memory version of skbs.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */
static inline void
vmxnet3_dev_kfree_skb(struct vmxnet3_adapter *adapter, struct sk_buff *skb)
{
	if (adapter->is_shm)
		vmxnet3_free_skbpages(adapter, skb);

compat_dev_kfree_skb(skb, FREE_WRITE);
}

static inline void
vmxnet3_dev_kfree_skb_any(struct vmxnet3_adapter *adapter, struct sk_buff *skb)
{
	if (adapter->is_shm)
		vmxnet3_free_skbpages(adapter, skb);

compat_dev_kfree_skb_any(skb, FREE_WRITE);
}

static inline void
vmxnet3_dev_kfree_skb_irq(struct vmxnet3_adapter *adapter, struct sk_buff *skb)
{
	if (adapter->is_shm)
		vmxnet3_free_skbpages(adapter, skb);

compat_dev_kfree_skb_irq(skb, FREE_WRITE);
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_skb_* --
 *
 *      Covers for (compat_)skb_*. Deal with the shared memory version of skbs.
 *
 * Results:
 *      Depends.
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */
static inline unsigned int
vmxnet3_skb_headlen(struct vmxnet3_adapter *adapter, struct sk_buff *skb)
{
	if (adapter->is_shm)
		return VMXNET3_SHM_SKB_GETLEN(skb);
	else
		return compat_skb_headlen(skb);

}

static inline void
vmxnet3_skb_put(struct vmxnet3_adapter *adapter, struct sk_buff *skb, unsigned int len)
{
	if (!adapter->is_shm) {
		skb_put(skb, len);
	} else {
		unsigned int oldlen = VMXNET3_SHM_SKB_GETLEN(skb);
		VMXNET3_SHM_SKB_SETLEN(skb, len + oldlen);
	}
}

static inline struct sk_buff*
vmxnet3_dev_alloc_skb(struct vmxnet3_adapter *adapter, unsigned long length)
{
	if (adapter->is_shm) {
		int idx;
		struct sk_buff* skb;
		idx = vmxnet3_shm_alloc_page(adapter->shm);
		if (idx == SHM_INVALID_IDX)
			return NULL;

/* The length is arbitrary because that memory shouldn't be used */
		skb = dev_alloc_skb(100);
		if (skb == NULL) {
			vmxnet3_shm_free_page(adapter->shm, idx);
			return NULL;
		}

VMXNET3_SHM_SKB_SETIDX(skb, idx);
		VMXNET3_SHM_SKB_SETLEN(skb, 0);

return skb;
	} else {
		return dev_alloc_skb(length);
	}
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_map_* --
 *
 *      Covers for pci_map_*. Deal with the shared memory version of skbs.
 *
 * Results:
 *      DMA address
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */
static inline dma_addr_t
vmxnet3_map_single(struct vmxnet3_adapter *adapter,
		struct sk_buff * skb,
		size_t offset,
		size_t len,
		int direction)
{
	if (adapter->is_shm) {
		unsigned long shm_idx = VMXNET3_SHM_SKB_GETIDX(skb);
		struct page *real_page = VMXNET3_SHM_IDX2PAGE(adapter->shm, shm_idx);
		return pci_map_page(adapter->pdev,
				real_page,
				offset,
				len,
				direction);
	} else {
		return pci_map_single(adapter->pdev,
				skb->data + offset,
				len,
				direction);
	}

}

static inline dma_addr_t
vmxnet3_map_page(struct vmxnet3_adapter *adapter,
		struct page *page,
		size_t offset,
		size_t len,
		int direction)
{
	if (adapter->is_shm) {
		unsigned long shm_idx = (unsigned long)page;
		page = VMXNET3_SHM_IDX2PAGE(adapter->shm, shm_idx);
	}

return pci_map_page(adapter->pdev,
			page,
			offset,
			len,
			direction);
}

/*
 *-----------------------------------------------------------------------------
 *
 * vmxnet3_(put|alloc)_page --
 *
 *      Allocation and release of pages. Either use regular or shared memory
 *      pages.
 *
 * Results:
 *      Depends
 *
 * Side effects:
 *      None.
 *
 *-----------------------------------------------------------------------------
 */
static inline void
vmxnet3_put_page(struct vmxnet3_adapter *adapter,
		struct page *page)
{
	if (!adapter->is_shm)
		put_page(page);
	else
		vmxnet3_shm_free_page(adapter->shm, (unsigned long)page);
}

static inline void *
vmxnet3_alloc_page(struct vmxnet3_adapter *adapter)
{
	if (adapter->is_shm)
		return (void*) (unsigned long) vmxnet3_shm_alloc_page(adapter->shm);
	else
		return alloc_page(GFP_ATOMIC);

}

#endif
������������������������������������������������������������������������������vmxnet3-only/vmxnet3_defs.h�������������������������������������������������������������������������0000444�0000000�0000000�00000070312�13432725340�014721� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/*
 * vmxnet3_defs.h --
 *
 *      Definitions shared by device emulation and guest drivers for
 *      VMXNET3 NIC
 */

#ifndef _VMXNET3_DEFS_H_
#define _VMXNET3_DEFS_H_

#define INCLUDE_ALLOW_USERLEVEL
#define INCLUDE_ALLOW_VMKERNEL
#define INCLUDE_ALLOW_DISTRIBUTE
#define INCLUDE_ALLOW_VMKDRIVERS
#define INCLUDE_ALLOW_VMCORE
#define INCLUDE_ALLOW_MODULE
#include "includeCheck.h"

#include "upt1_defs.h"

/* all registers are 32 bit wide */
/* BAR 1 */
#define VMXNET3_REG_VRRS  0x0    /* Vmxnet3 Revision Report Selection */
#define VMXNET3_REG_UVRS  0x8    /* UPT Version Report Selection */
#define VMXNET3_REG_DSAL  0x10   /* Driver Shared Address Low */
#define VMXNET3_REG_DSAH  0x18   /* Driver Shared Address High */
#define VMXNET3_REG_CMD   0x20   /* Command */
#define VMXNET3_REG_MACL  0x28   /* MAC Address Low */
#define VMXNET3_REG_MACH  0x30   /* MAC Address High */
#define VMXNET3_REG_ICR   0x38   /* Interrupt Cause Register */
#define VMXNET3_REG_ECR   0x40   /* Event Cause Register */

#define VMXNET3_REG_WSAL  0xF00  /* Wireless Shared Address Lo  */
#define VMXNET3_REG_WSAH  0xF08  /* Wireless Shared Address Hi  */
#define VMXNET3_REG_WCMD  0xF18  /* Wireless Command */

/* BAR 0 */
#define VMXNET3_REG_IMR      0x0   /* Interrupt Mask Register */
#define VMXNET3_REG_TXPROD   0x600 /* Tx Producer Index */
#define VMXNET3_REG_RXPROD   0x800 /* Rx Producer Index for ring 1 */
#define VMXNET3_REG_RXPROD2  0xA00 /* Rx Producer Index for ring 2 */

#define VMXNET3_PT_REG_SIZE     4096    /* BAR 0 */
#define VMXNET3_VD_REG_SIZE     4096    /* BAR 1 */

/*
 * The two Vmxnet3 MMIO Register PCI BARs (BAR 0 at offset 10h and BAR 1 at
 * offset 14h)  as well as the MSI-X BAR are combined into one PhysMem region:
 * <-VMXNET3_PT_REG_SIZE-><-VMXNET3_VD_REG_SIZE-><-VMXNET3_MSIX_BAR_SIZE-->
 * -------------------------------------------------------------------------
 * |Pass Thru Registers  | Virtual Dev Registers | MSI-X Vector/PBA Table  |
 * -------------------------------------------------------------------------
 * VMXNET3_MSIX_BAR_SIZE is defined in "vmxnet3Int.h"
 */
#define VMXNET3_PHYSMEM_PAGES   4

#define VMXNET3_REG_ALIGN       8  /* All registers are 8-byte aligned. */
#define VMXNET3_REG_ALIGN_MASK  0x7

/* I/O Mapped access to registers */
#define VMXNET3_IO_TYPE_PT              0
#define VMXNET3_IO_TYPE_VD              1
#define VMXNET3_IO_ADDR(type, reg)      (((type) << 24) | ((reg) & 0xFFFFFF))
#define VMXNET3_IO_TYPE(addr)           ((addr) >> 24)
#define VMXNET3_IO_REG(addr)            ((addr) & 0xFFFFFF)

#ifndef __le16
#define __le16 uint16
#endif
#ifndef __le32
#define __le32 uint32
#endif
#ifndef __le64
#define __le64 uint64
#endif

typedef enum {
   VMXNET3_CMD_FIRST_SET = 0xCAFE0000,
   VMXNET3_CMD_ACTIVATE_DEV = VMXNET3_CMD_FIRST_SET,
   VMXNET3_CMD_QUIESCE_DEV,
   VMXNET3_CMD_RESET_DEV,
   VMXNET3_CMD_UPDATE_RX_MODE,
   VMXNET3_CMD_UPDATE_MAC_FILTERS,
   VMXNET3_CMD_UPDATE_VLAN_FILTERS,
   VMXNET3_CMD_UPDATE_RSSIDT,
   VMXNET3_CMD_UPDATE_IML,
   VMXNET3_CMD_UPDATE_PMCFG,
   VMXNET3_CMD_UPDATE_FEATURE,
   VMXNET3_CMD_STOP_EMULATION,
   VMXNET3_CMD_LOAD_PLUGIN,
   VMXNET3_CMD_ACTIVATE_VF,
   VMXNET3_CMD_SET_POLLING,
   VMXNET3_CMD_SET_COALESCE,
   VMXNET3_CMD_REGISTER_MEMREGS,
   VMXNET3_CMD_SET_RSS_FIELDS,

VMXNET3_CMD_FIRST_GET = 0xF00D0000,
   VMXNET3_CMD_GET_QUEUE_STATUS = VMXNET3_CMD_FIRST_GET,
   VMXNET3_CMD_GET_STATS,
   VMXNET3_CMD_GET_LINK,
   VMXNET3_CMD_GET_PERM_MAC_LO,
   VMXNET3_CMD_GET_PERM_MAC_HI,
   VMXNET3_CMD_GET_DID_LO,
   VMXNET3_CMD_GET_DID_HI,
   VMXNET3_CMD_GET_DEV_EXTRA_INFO,
   VMXNET3_CMD_GET_CONF_INTR,
   VMXNET3_CMD_GET_ADAPTIVE_RING_INFO,
   VMXNET3_CMD_GET_TXDATA_DESC_SIZE,
   VMXNET3_CMD_GET_COALESCE,
   VMXNET3_CMD_GET_RSS_FIELDS,
   VMXNET3_CMD_GET_ENCAP_DSTPORT,
} Vmxnet3_Cmd;

/* Adaptive Ring Info Flags */
#define VMXNET3_DISABLE_ADAPTIVE_RING 1

/*
 *	Little Endian layout of bitfields -
 *	Byte 0 :	7.....len.....0
 *	Byte 1 :	oco gen 13.len.8
 *	Byte 2 : 	5.msscof.0 ext1  dtype
 *	Byte 3 : 	13...msscof...6
 *
 *	Big Endian layout of bitfields -
 *	Byte 0:		13...msscof...6
 *	Byte 1 : 	5.msscof.0 ext1  dtype
 *	Byte 2 :	oco gen 13.len.8
 *	Byte 3 :	7.....len.....0
 *
 *	Thus, le32_to_cpu on the dword will allow the big endian driver to read
 *	the bit fields correctly. And cpu_to_le32 will convert bitfields
 *	bit fields written by big endian driver to format required by device.
 */

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_TxDesc {
   __le64 addr;

#ifdef __BIG_ENDIAN_BITFIELD
   uint32 msscof:14;  /* MSS, checksum offset, flags */
   uint32 ext1:1;
   uint32 dtype:1;    /* descriptor type */
   uint32 oco:1;      /* Outer csum offload */
   uint32 gen:1;      /* generation bit */
   uint32 len:14;
#else
   uint32 len:14;
   uint32 gen:1;      /* generation bit */
   uint32 oco:1;      /* Outer csum offload */
   uint32 dtype:1;    /* descriptor type */
   uint32 ext1:1;
   uint32 msscof:14;  /* MSS, checksum offset, flags */
#endif  /* __BIG_ENDIAN_BITFIELD */

#ifdef __BIG_ENDIAN_BITFIELD
   uint32 tci:16;     /* Tag to Insert */
   uint32 ti:1;       /* VLAN Tag Insertion */
   uint32 ext2:1;
   uint32 cq:1;       /* completion request */
   uint32 eop:1;      /* End Of Packet */
   uint32 om:2;       /* offload mode */
   uint32 hlen:10;    /* header len */
#else
   uint32 hlen:10;    /* header len */
   uint32 om:2;       /* offload mode */
   uint32 eop:1;      /* End Of Packet */
   uint32 cq:1;       /* completion request */
   uint32 ext2:1;
   uint32 ti:1;       /* VLAN Tag Insertion */
   uint32 tci:16;     /* Tag to Insert */
#endif  /* __BIG_ENDIAN_BITFIELD */
}
#include "vmware_pack_end.h"
Vmxnet3_TxDesc;

/* TxDesc.OM values */
#define VMXNET3_OM_NONE  0          /* 0b00 */
#define VMXNET3_OM_ENCAP 1          /* 0b01 */
#define VMXNET3_OM_CSUM  2          /* 0b10 */
#define VMXNET3_OM_TSO   3          /* 0b11 */

/* fields in TxDesc we access w/o using bit fields */
#define VMXNET3_TXD_EOP_SHIFT 12
#define VMXNET3_TXD_CQ_SHIFT  13
#define VMXNET3_TXD_GEN_SHIFT 14
#define VMXNET3_TXD_EOP_DWORD_SHIFT 3
#define VMXNET3_TXD_GEN_DWORD_SHIFT 2

#define VMXNET3_TXD_CQ  (1 << VMXNET3_TXD_CQ_SHIFT)
#define VMXNET3_TXD_EOP (1 << VMXNET3_TXD_EOP_SHIFT)
#define VMXNET3_TXD_GEN (1 << VMXNET3_TXD_GEN_SHIFT)

#define VMXNET3_TXD_GEN_SIZE 1
#define VMXNET3_TXD_EOP_SIZE 1

#define VMXNET3_HDR_COPY_SIZE   128
typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_TxDataDesc {
   uint8 data[VMXNET3_HDR_COPY_SIZE];
}
#include "vmware_pack_end.h"
Vmxnet3_TxDataDesc;
typedef uint8 Vmxnet3_RxDataDesc;

#define VMXNET3_TCD_GEN_SHIFT	31
#define VMXNET3_TCD_GEN_SIZE	1
#define VMXNET3_TCD_TXIDX_SHIFT	0
#define VMXNET3_TCD_TXIDX_SIZE	12
#define VMXNET3_TCD_GEN_DWORD_SHIFT	3

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_TxCompDesc {
   uint32 txdIdx:12;    /* Index of the EOP TxDesc */
   uint32 ext1:20;

__le32 ext2;
   __le32 ext3;

uint32 rsvd:24;
   uint32 type:7;       /* completion type */
   uint32 gen:1;        /* generation bit */
}
#include "vmware_pack_end.h"
Vmxnet3_TxCompDesc;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_RxDesc {
   __le64 addr;

#ifdef __BIG_ENDIAN_BITFIELD
   uint32 gen:1;        /* Generation bit */
   uint32 rsvd:15;
   uint32 dtype:1;      /* Descriptor type */
   uint32 btype:1;      /* Buffer Type */
   uint32 len:14;
#else
   uint32 len:14;
   uint32 btype:1;      /* Buffer Type */
   uint32 dtype:1;      /* Descriptor type */
   uint32 rsvd:15;
   uint32 gen:1;        /* Generation bit */
#endif
   __le32 ext1;
}
#include "vmware_pack_end.h"
Vmxnet3_RxDesc;

/* values of RXD.BTYPE */
#define VMXNET3_RXD_BTYPE_HEAD   0    /* head only */
#define VMXNET3_RXD_BTYPE_BODY   1    /* body only */

/* fields in RxDesc we access w/o using bit fields */
#define VMXNET3_RXD_BTYPE_SHIFT  14
#define VMXNET3_RXD_GEN_SHIFT    31

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_RxCompDesc {
#ifdef __BIG_ENDIAN_BITFIELD
   uint32 ext2:1;
   uint32 cnc:1;        /* Checksum Not Calculated */
   uint32 rssType:4;    /* RSS hash type used */
   uint32 rqID:10;      /* rx queue/ring ID */
   uint32 sop:1;        /* Start of Packet */
   uint32 eop:1;        /* End of Packet */
   uint32 ext1:2;
   uint32 rxdIdx:12;    /* Index of the RxDesc */
#else
   uint32 rxdIdx:12;    /* Index of the RxDesc */
   uint32 ext1:2;
   uint32 eop:1;        /* End of Packet */
   uint32 sop:1;        /* Start of Packet */
   uint32 rqID:10;      /* rx queue/ring ID */
   uint32 rssType:4;    /* RSS hash type used */
   uint32 cnc:1;        /* Checksum Not Calculated */
   uint32 ext2:1;
#endif  /* __BIG_ENDIAN_BITFIELD */

__le32 rssHash;      /* RSS hash value */

#ifdef __BIG_ENDIAN_BITFIELD
   uint32 tci:16;       /* Tag stripped */
   uint32 ts:1;         /* Tag is stripped */
   uint32 err:1;        /* Error */
   uint32 len:14;       /* data length */
#else
   uint32 len:14;       /* data length */
   uint32 err:1;        /* Error */
   uint32 ts:1;         /* Tag is stripped */
   uint32 tci:16;       /* Tag stripped */
#endif  /* __BIG_ENDIAN_BITFIELD */

#ifdef __BIG_ENDIAN_BITFIELD
   uint32 gen:1;        /* generation bit */
   uint32 type:7;       /* completion type */
   uint32 fcs:1;        /* Frame CRC correct */
   uint32 frg:1;        /* IP Fragment */
   uint32 v4:1;         /* IPv4 */
   uint32 v6:1;         /* IPv6 */
   uint32 ipc:1;        /* IP Checksum Correct */
   uint32 tcp:1;        /* TCP packet */
   uint32 udp:1;        /* UDP packet */
   uint32 tuc:1;        /* TCP/UDP Checksum Correct */
   uint32 csum:16;
#else
   uint32 csum:16;
   uint32 tuc:1;        /* TCP/UDP Checksum Correct */
   uint32 udp:1;        /* UDP packet */
   uint32 tcp:1;        /* TCP packet */
   uint32 ipc:1;        /* IP Checksum Correct */
   uint32 v6:1;         /* IPv6 */
   uint32 v4:1;         /* IPv4 */
   uint32 frg:1;        /* IP Fragment */
   uint32 fcs:1;        /* Frame CRC correct */
   uint32 type:7;       /* completion type */
   uint32 gen:1;        /* generation bit */
#endif  /* __BIG_ENDIAN_BITFIELD */
}
#include "vmware_pack_end.h"
Vmxnet3_RxCompDesc;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_RxCompDescExt {
   __le32 dword1;
   uint8  segCnt;       /* Number of aggregated packets */
   uint8  dupAckCnt;    /* Number of duplicate Acks */
   __le16 tsDelta;      /* TCP timestamp difference */

#ifdef __BIG_ENDIAN_BITFIELD
   uint32 encap:1;      /* LRO info refers to inner pkt */
   uint32 reserved:31;
#else
   uint32 reserved:31;
   uint32 encap:1;      /* LRO info refers to inner pkt */
#endif  /* __BIG_ENDIAN_BITFIELD */

#ifdef __BIG_ENDIAN_BITFIELD
   uint32 gen:1;        /* generation bit */
   uint32 type:7;       /* completion type */
   uint32 fcs:1;        /* Frame CRC correct */
   uint32 frg:1;        /* IP Fragment */
   uint32 v4:1;         /* IPv4 */
   uint32 v6:1;         /* IPv6 */
   uint32 ipc:1;        /* IP Checksum Correct */
   uint32 tcp:1;        /* TCP packet */
   uint32 udp:1;        /* UDP packet */
   uint32 tuc:1;        /* TCP/UDP Checksum Correct */
   uint32 mss:16;
#else
   uint32 mss:16;
   uint32 tuc:1;        /* TCP/UDP Checksum Correct */
   uint32 udp:1;        /* UDP packet */
   uint32 tcp:1;        /* TCP packet */
   uint32 ipc:1;        /* IP Checksum Correct */
   uint32 v6:1;         /* IPv6 */
   uint32 v4:1;         /* IPv4 */
   uint32 frg:1;        /* IP Fragment */
   uint32 fcs:1;        /* Frame CRC correct */
   uint32 type:7;       /* completion type */
   uint32 gen:1;        /* generation bit */
#endif  /* __BIG_ENDIAN_BITFIELD */
}
#include "vmware_pack_end.h"
Vmxnet3_RxCompDescExt;

/* fields in RxCompDesc we access via Vmxnet3_GenericDesc.dword[3] */
#define VMXNET3_RCD_TUC_SHIFT  16
#define VMXNET3_RCD_IPC_SHIFT  19

/* fields in RxCompDesc we access via Vmxnet3_GenericDesc.qword[1] */
#define VMXNET3_RCD_TYPE_SHIFT 56
#define VMXNET3_RCD_GEN_SHIFT  63

/* csum OK for TCP/UDP pkts over IP */
#define VMXNET3_RCD_CSUM_OK (1 << VMXNET3_RCD_TUC_SHIFT | 1 << VMXNET3_RCD_IPC_SHIFT)

/* value of RxCompDesc.rssType */
#define VMXNET3_RCD_RSS_TYPE_NONE     0
#define VMXNET3_RCD_RSS_TYPE_IPV4     1
#define VMXNET3_RCD_RSS_TYPE_TCPIPV4  2
#define VMXNET3_RCD_RSS_TYPE_IPV6     3
#define VMXNET3_RCD_RSS_TYPE_TCPIPV6  4
#define VMXNET3_RCD_RSS_TYPE_UDPIPV4  5
#define VMXNET3_RCD_RSS_TYPE_UDPIPV6  6
#define VMXNET3_RCD_RSS_TYPE_ESPIPV4  7
#define VMXNET3_RCD_RSS_TYPE_ESPIPV6  8

/* a union for accessing all cmd/completion descriptors */
typedef union Vmxnet3_GenericDesc {
   __le64                qword[2];
   __le32                dword[4];
   __le16                word[8];
   Vmxnet3_TxDesc        txd;
   Vmxnet3_RxDesc        rxd;
   Vmxnet3_TxCompDesc    tcd;
   Vmxnet3_RxCompDesc    rcd;
   Vmxnet3_RxCompDescExt rcdExt;
} Vmxnet3_GenericDesc;

#define VMXNET3_INIT_GEN       1

#define VMXNET3_INVALID_QUEUEID -1

/* Max size of a single tx buffer */
#define VMXNET3_MAX_TX_BUF_SIZE  (1 << 14)

/* # of tx desc needed for a tx buffer size */
#define VMXNET3_TXD_NEEDED(size) (((size) + VMXNET3_MAX_TX_BUF_SIZE - 1) / VMXNET3_MAX_TX_BUF_SIZE)

/* max # of tx descs for a non-tso pkt */
#define VMXNET3_MAX_TXD_PER_PKT 16

/* Max size of a single rx buffer */
#define VMXNET3_MAX_RX_BUF_SIZE  ((1 << 14) - 1)
/* Minimum size of a type 0 buffer */
#define VMXNET3_MIN_T0_BUF_SIZE  128
#define VMXNET3_MAX_CSUM_OFFSET  1024

/* Ring base address alignment */
#define VMXNET3_RING_BA_ALIGN   512
#define VMXNET3_RING_BA_MASK    (VMXNET3_RING_BA_ALIGN - 1)

/* Ring size must be a multiple of 32 */
#define VMXNET3_RING_SIZE_ALIGN 32
#define VMXNET3_RING_SIZE_MASK  (VMXNET3_RING_SIZE_ALIGN - 1)

/* Rx Data Ring buffer size must be a multiple of 64 bytes */
#define VMXNET3_RXDATA_DESC_SIZE_ALIGN 64
#define VMXNET3_RXDATA_DESC_SIZE_MASK  (VMXNET3_RXDATA_DESC_SIZE_ALIGN - 1)

/* Tx Data Ring buffer size must be a multiple of 64 bytes */
#define VMXNET3_TXDATA_DESC_SIZE_ALIGN 64
#define VMXNET3_TXDATA_DESC_SIZE_MASK  (VMXNET3_TXDATA_DESC_SIZE_ALIGN - 1)

/* Max ring size */
#define VMXNET3_TX_RING_MAX_SIZE   4096
#define VMXNET3_TC_RING_MAX_SIZE   4096
#define VMXNET3_RX_RING_MAX_SIZE   4096
#define VMXNET3_RX_RING2_MAX_SIZE  4096
#define VMXNET3_RC_RING_MAX_SIZE   8192

/* Large enough to accommodate typical payload + encap + extension header */
#define VMXNET3_RXDATA_DESC_MAX_SIZE   2048
#define VMXNET3_TXDATA_DESC_MIN_SIZE   128
#define VMXNET3_TXDATA_DESC_MAX_SIZE   2048

/* a list of reasons for queue stop */

#define VMXNET3_ERR_NOEOP        0x80000000  /* cannot find the EOP desc of a pkt */
#define VMXNET3_ERR_TXD_REUSE    0x80000001  /* reuse a TxDesc before tx completion */
#define VMXNET3_ERR_BIG_PKT      0x80000002  /* too many TxDesc for a pkt */
#define VMXNET3_ERR_DESC_NOT_SPT 0x80000003  /* descriptor type not supported */
#define VMXNET3_ERR_SMALL_BUF    0x80000004  /* type 0 buffer too small */
#define VMXNET3_ERR_STRESS       0x80000005  /* stress option firing in vmkernel */
#define VMXNET3_ERR_SWITCH       0x80000006  /* mode switch failure */
#define VMXNET3_ERR_TXD_INVALID  0x80000007  /* invalid TxDesc */

/* completion descriptor types */
#define VMXNET3_CDTYPE_TXCOMP      0    /* Tx Completion Descriptor */
#define VMXNET3_CDTYPE_RXCOMP      3    /* Rx Completion Descriptor */
#define VMXNET3_CDTYPE_RXCOMP_LRO  4    /* Rx Completion Descriptor for LRO */

#define VMXNET3_GOS_BITS_UNK    0   /* unknown */
#define VMXNET3_GOS_BITS_32     1
#define VMXNET3_GOS_BITS_64     2

#define VMXNET3_GOS_TYPE_UNK        0 /* unknown */
#define VMXNET3_GOS_TYPE_LINUX      1
#define VMXNET3_GOS_TYPE_WIN        2
#define VMXNET3_GOS_TYPE_SOLARIS    3
#define VMXNET3_GOS_TYPE_FREEBSD    4
#define VMXNET3_GOS_TYPE_PXE        5

/* All structures in DriverShared are padded to multiples of 8 bytes */

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_GOSInfo {
#ifdef __BIG_ENDIAN_BITFIELD
   uint32   gosMisc: 10;    /* other info about gos */
   uint32   gosVer:  16;    /* gos version */
   uint32   gosType: 4;     /* which guest */
   uint32   gosBits: 2;     /* 32-bit or 64-bit? */
#else
   uint32   gosBits: 2;     /* 32-bit or 64-bit? */
   uint32   gosType: 4;     /* which guest */
   uint32   gosVer:  16;    /* gos version */
   uint32   gosMisc: 10;    /* other info about gos */
#endif  /* __BIG_ENDIAN_BITFIELD */
}
#include "vmware_pack_end.h"
Vmxnet3_GOSInfo;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_DriverInfo {
   __le32          version;        /* driver version */
   Vmxnet3_GOSInfo gos;
   __le32          vmxnet3RevSpt;  /* vmxnet3 revision supported */
   __le32          uptVerSpt;      /* upt version supported */
}
#include "vmware_pack_end.h"
Vmxnet3_DriverInfo;

#define VMXNET3_REV1_MAGIC  0xbabefee1

/* 
 * QueueDescPA must be 128 bytes aligned. It points to an array of
 * Vmxnet3_TxQueueDesc followed by an array of Vmxnet3_RxQueueDesc.
 * The number of Vmxnet3_TxQueueDesc/Vmxnet3_RxQueueDesc are specified by
 * Vmxnet3_MiscConf.numTxQueues/numRxQueues, respectively.
 */
#define VMXNET3_QUEUE_DESC_ALIGN  128

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_MiscConf {
   Vmxnet3_DriverInfo driverInfo;
   __le64             uptFeatures;
   __le64             ddPA;         /* driver data PA */
   __le64             queueDescPA;  /* queue descriptor table PA */
   __le32             ddLen;        /* driver data len */
   __le32             queueDescLen; /* queue descriptor table len, in bytes */
   __le32             mtu;
   __le16             maxNumRxSG;
   uint8              numTxQueues;
   uint8              numRxQueues;
   __le32             reserved[4];
}
#include "vmware_pack_end.h"
Vmxnet3_MiscConf;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_TxQueueConf {
   __le64    txRingBasePA;
   __le64    dataRingBasePA;
   __le64    compRingBasePA;
   __le64    ddPA;         /* driver data */
   __le64    reserved;
   __le32    txRingSize;   /* # of tx desc */
   __le32    dataRingSize; /* # of data desc */
   __le32    compRingSize; /* # of comp desc */
   __le32    ddLen;        /* size of driver data */
   uint8     intrIdx;
   uint8     _pad1[1];
   __le16    txDataRingDescSize;
   uint8     _pad2[4];
}
#include "vmware_pack_end.h"
Vmxnet3_TxQueueConf;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_RxQueueConf {
   __le64    rxRingBasePA[2];
   __le64    compRingBasePA;
   __le64    ddPA;            /* driver data */
   __le64    rxDataRingBasePA;
   __le32    rxRingSize[2];   /* # of rx desc */
   __le32    compRingSize;    /* # of rx comp desc */
   __le32    ddLen;           /* size of driver data */
   uint8     intrIdx;
   uint8     _pad1[1];
   __le16    rxDataRingDescSize; /* size of rx data ring buffer */
   uint8     _pad2[4];
}
#include "vmware_pack_end.h"
Vmxnet3_RxQueueConf;

enum vmxnet3_intr_mask_mode {
   VMXNET3_IMM_AUTO   = 0,
   VMXNET3_IMM_ACTIVE = 1,
   VMXNET3_IMM_LAZY   = 2
};

enum vmxnet3_intr_type {
   VMXNET3_IT_AUTO = 0,
   VMXNET3_IT_INTX = 1,
   VMXNET3_IT_MSI  = 2,
   VMXNET3_IT_MSIX = 3
};

#define VMXNET3_MAX_TX_QUEUES  8
#define VMXNET3_MAX_RX_QUEUES  16
/* addition 1 for events */
#define VMXNET3_MAX_INTRS      25

/* value of intrCtrl */
#define VMXNET3_IC_DISABLE_ALL  0x1   /* bit 0 */

#define VMXNET3_COAL_STATIC_MAX_DEPTH        128
#define VMXNET3_COAL_RBC_MIN_RATE            100
#define VMXNET3_COAL_RBC_MAX_RATE            100000

enum Vmxnet3_CoalesceMode {
   VMXNET3_COALESCE_DISABLED   = 0,
   VMXNET3_COALESCE_ADAPT      = 1,
   VMXNET3_COALESCE_STATIC     = 2,
   VMXNET3_COALESCE_RBC        = 3
};

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_CoalesceRbc {
   uint32 rbc_rate;
}
#include "vmware_pack_end.h"
Vmxnet3_CoalesceRbc;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_CoalesceStatic {
   uint32 tx_depth;
   uint32 tx_comp_depth;
   uint32 rx_depth;
}
#include "vmware_pack_end.h"
Vmxnet3_CoalesceStatic;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_CoalesceScheme {
   enum Vmxnet3_CoalesceMode coalMode;
   union {
      Vmxnet3_CoalesceRbc    coalRbc;
      Vmxnet3_CoalesceStatic coalStatic;
   } coalPara;
}
#include "vmware_pack_end.h"
Vmxnet3_CoalesceScheme;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_IntrConf {
   Bool   autoMask;
   uint8  numIntrs;      /* # of interrupts */
   uint8  eventIntrIdx;
   uint8  modLevels[VMXNET3_MAX_INTRS]; /* moderation level for each intr */
   __le32 intrCtrl;
   __le32 reserved[2];
}
#include "vmware_pack_end.h"
Vmxnet3_IntrConf;

/* one bit per VLAN ID, the size is in the units of uint32 */
#define VMXNET3_VFT_SIZE  (4096 / (sizeof(uint32) * 8))

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_QueueStatus {
   Bool    stopped;
   uint8   _pad[3];
   __le32  error;
}
#include "vmware_pack_end.h"
Vmxnet3_QueueStatus;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_TxQueueCtrl {
   __le32  txNumDeferred;
   __le32  txThreshold;
   __le64  reserved;
}
#include "vmware_pack_end.h"
Vmxnet3_TxQueueCtrl;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_RxQueueCtrl {
   Bool    updateRxProd;
   uint8   _pad[7];
   __le64  reserved;
}
#include "vmware_pack_end.h"
Vmxnet3_RxQueueCtrl;

#define VMXNET3_RXM_UCAST     0x01  /* unicast only */
#define VMXNET3_RXM_MCAST     0x02  /* multicast passing the filters */
#define VMXNET3_RXM_BCAST     0x04  /* broadcast only */
#define VMXNET3_RXM_ALL_MULTI 0x08  /* all multicast */
#define VMXNET3_RXM_PROMISC   0x10  /* promiscuous */

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_RxFilterConf {
   __le32   rxMode;       /* VMXNET3_RXM_xxx */
   __le16   mfTableLen;   /* size of the multicast filter table */
   __le16   _pad1;
   __le64   mfTablePA;    /* PA of the multicast filters table */
   __le32   vfTable[VMXNET3_VFT_SIZE]; /* vlan filter */
}
#include "vmware_pack_end.h"
Vmxnet3_RxFilterConf;

#define VMXNET3_PM_MAX_FILTERS        6
#define VMXNET3_PM_MAX_PATTERN_SIZE   128
#define VMXNET3_PM_MAX_MASK_SIZE      (VMXNET3_PM_MAX_PATTERN_SIZE / 8)

#define VMXNET3_PM_WAKEUP_MAGIC       0x01  /* wake up on magic pkts */
#define VMXNET3_PM_WAKEUP_FILTER      0x02  /* wake up on pkts matching filters */

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_PM_PktFilter {
   uint8 maskSize;
   uint8 patternSize;
   uint8 mask[VMXNET3_PM_MAX_MASK_SIZE];
   uint8 pattern[VMXNET3_PM_MAX_PATTERN_SIZE];
   uint8 pad[6];
}
#include "vmware_pack_end.h"
Vmxnet3_PM_PktFilter;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_PMConf {
   __le16               wakeUpEvents;  /* VMXNET3_PM_WAKEUP_xxx */
   uint8                numFilters;
   uint8                pad[5];
   Vmxnet3_PM_PktFilter filters[VMXNET3_PM_MAX_FILTERS];
}
#include "vmware_pack_end.h"
Vmxnet3_PMConf;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_VariableLenConfDesc {
   __le32              confVer;
   __le32              confLen;
   __le64              confPA;
}
#include "vmware_pack_end.h"
Vmxnet3_VariableLenConfDesc;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_DSDevRead {
   /* read-only region for device, read by dev in response to a SET cmd */
   Vmxnet3_MiscConf     misc;
   Vmxnet3_IntrConf     intrConf;
   Vmxnet3_RxFilterConf rxFilterConf;
   Vmxnet3_VariableLenConfDesc  rssConfDesc;
   Vmxnet3_VariableLenConfDesc  pmConfDesc;
   Vmxnet3_VariableLenConfDesc  pluginConfDesc;
}
#include "vmware_pack_end.h"
Vmxnet3_DSDevRead;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_TxQueueDesc {
   Vmxnet3_TxQueueCtrl ctrl;
   Vmxnet3_TxQueueConf conf;
   /* Driver read after a GET command */
   Vmxnet3_QueueStatus status;
   UPT1_TxStats        stats;
   uint8               _pad[88]; /* 128 aligned */
}
#include "vmware_pack_end.h"
Vmxnet3_TxQueueDesc;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_RxQueueDesc {
   Vmxnet3_RxQueueCtrl ctrl;
   Vmxnet3_RxQueueConf conf;
   /* Driver read after a GET command */
   Vmxnet3_QueueStatus status;
   UPT1_RxStats        stats;
   uint8               _pad[88]; /* 128 aligned */
}
#include "vmware_pack_end.h"
Vmxnet3_RxQueueDesc;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_SetPolling {
   uint8               enablePolling;
}
#include "vmware_pack_end.h"
Vmxnet3_SetPolling;

typedef
#include "vmware_pack_begin.h"

struct Vmxnet3_MemoryRegion {
   __le64            startPA;  // starting physical address
   __le32            length;   // limit the length to be less than 4G
   /*
    * any bits is set in txQueueBits or rxQueueBits indicate this region
    * is applicable for the relevant queue
    */
   __le16            txQueueBits; // bit n corresponding to tx queue n
   __le16            rxQueueBits; // bit n corresponding to rx queueb n
}
#include "vmware_pack_end.h"
Vmxnet3_MemoryRegion;

/*
 * Assume each queue can have upto 16 memory region
 * we have 8 + 8 = 16 queues. So max regions is
 * defined as 16 * 16
 * when more region is passed to backend, the handling
 * is undefined, Backend can choose to fail the the request
 * or ignore the extra region.
 */
#define MAX_MEMORY_REGION_PER_QUEUE 16
#define MAX_MEMORY_REGION_PER_DEVICE (16 * 16)

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_MemRegs {
   __le16           numRegs;
   __le16           pad[3];
   Vmxnet3_MemoryRegion memRegs[1];
}
#include "vmware_pack_end.h"
Vmxnet3_MemRegs;

typedef enum Vmxnet3_RSSField {
   VMXNET3_RSS_FIELDS_TCPIP4 = 0x0001,
   VMXNET3_RSS_FIELDS_TCPIP6 = 0x0002,
   VMXNET3_RSS_FIELDS_UDPIP4 = 0x0004,
   VMXNET3_RSS_FIELDS_UDPIP6 = 0x0008,
   VMXNET3_RSS_FIELDS_ESPIP4 = 0x0010,
   VMXNET3_RSS_FIELDS_ESPIP6 = 0x0020,
} Vmxnet3_RSSField;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_EncapDstPort {
   __le16            geneveDstPort;
   __le16            vxlanDstPort;
}
#include "vmware_pack_end.h"
Vmxnet3_EncapDstPort;

/*
 * If a command data does not exceed 16 bytes, it can use
 * the shared memory directly. Otherwise use variable length
 * configuration descriptor to pass the data.
 */
typedef
#include "vmware_pack_begin.h"
union Vmxnet3_CmdInfo {
   Vmxnet3_VariableLenConfDesc varConf;
   Vmxnet3_SetPolling          setPolling;
   Vmxnet3_RSSField            setRSSFields;
   Vmxnet3_EncapDstPort        encapDstPort;
   __le64                      data[2];
}
#include "vmware_pack_end.h"
Vmxnet3_CmdInfo;

typedef
#include "vmware_pack_begin.h"
struct Vmxnet3_DriverShared {
   __le32               magic;
   __le32               pad; /* make devRead start at 64-bit boundaries */
   Vmxnet3_DSDevRead    devRead;
   __le32               ecr;
   __le32               reserved;
   union {
      __le32            reserved1[4];
      Vmxnet3_CmdInfo   cmdInfo; /* only valid in the context of executing the relevant
                                  * command.
                                  */
   } cu;
}
#include "vmware_pack_end.h"
Vmxnet3_DriverShared;

#define VMXNET3_ECR_RQERR       (1 << 0)
#define VMXNET3_ECR_TQERR       (1 << 1)
#define VMXNET3_ECR_LINK        (1 << 2)
#define VMXNET3_ECR_DIC         (1 << 3)
#define VMXNET3_ECR_DEBUG       (1 << 4)

/* flip the gen bit of a ring */
#define VMXNET3_FLIP_RING_GEN(gen) ((gen) = (gen) ^ 0x1)

/* only use this if moving the idx won't affect the gen bit */
#define VMXNET3_INC_RING_IDX_ONLY(idx, ring_size) \
do {\
   (idx)++;\
   if (UNLIKELY((idx) == (ring_size))) {\
      (idx) = 0;\
   }\
} while (0)

#define VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid) \
   (vfTable)[(vid) >> 5] |= (1 << ((vid) & 31))
#define VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid) \
   (vfTable)[(vid) >> 5] &= ~(1 << ((vid) & 31))

#define VMXNET3_VFTABLE_ENTRY_IS_SET(vfTable, vid) \
   (((vfTable)[(vid) >> 5] & (1 << ((vid) & 31))) != 0)

#define VMXNET3_MAX_MTU     9000
#define VMXNET3_MIN_MTU     60

#define VMXNET3_LINK_UP         (10000 << 16 | 1)    // 10 Gbps, up
#define VMXNET3_LINK_DOWN       0

#define VMXWIFI_DRIVER_SHARED_LEN 8192

#define VMXNET3_DID_PASSTHRU    0xFFFF

#endif /* _VMXNET3_DEFS_H_ */
����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/Makefile.kernel������������������������������������������������������������������������0000444�0000000�0000000�00000003056�13432725331�015063� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������#!/usr/bin/make -f
##########################################################
# Copyright (C) 2008 VMware, Inc. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation version 2 and no later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
#
##########################################################

####
####  VMware vmxnet3 Makefile to be distributed externally
####

INCLUDE += -I.

ifdef OVT_SOURCE_DIR
INCLUDE += -I$(OVT_SOURCE_DIR)/modules/shared/vmxnet
endif

EXTRA_CFLAGS := $(CC_OPTS) $(INCLUDE)

EXTRA_CFLAGS += $(call vm_check_build, $(AUTOCONF_DIR)/skblin.c, -DVMW_SKB_LINEARIZE_2618, )
EXTRA_CFLAGS += $(call vm_check_build, $(AUTOCONF_DIR)/netif_num_params.c, , -DVMW_NETIF_SINGLE_NAPI_PARM)

obj-m += $(DRIVER).o

$(DRIVER)-y := $(subst $(SRCROOT)/, , $(patsubst %.c, %.o, $(wildcard $(SRCROOT)/*.c)))

clean:
	rm -rf $(wildcard $(DRIVER).mod.c $(DRIVER).ko .tmp_versions \
	       Module.symvers Modules.symvers Module.markers modules.order \
	       $(foreach dir,./,$(addprefix $(dir),.*.cmd .*.o.flags *.o)))
����������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������vmxnet3-only/upt1_defs.h����������������������������������������������������������������������������0000444�0000000�0000000�00000006463�13432725340�014214� 0����������������������������������������������������������������������������������������������������ustar  �root����������������������������root�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������/*********************************************************
 * Copyright (C) 2007-2018 VMware, Inc. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation version 2 and no later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 *
 *********************************************************/

/* upt1_defs.h
 *
 *      Definitions for UPTv1
 *
 *      Some of the defs are duplicated in vmkapi_net_upt.h, because
 *      vmkapi_net_upt.h cannot distribute with OSS yet and vmkapi headers can
 *      only include vmkapi headers. Make sure they are kept in sync!
 */

#ifndef _UPT1_DEFS_H
#define _UPT1_DEFS_H

#define UPT1_MAX_TX_QUEUES  64
#define UPT1_MAX_RX_QUEUES  64

#define UPT1_MAX_INTRS  (UPT1_MAX_TX_QUEUES + UPT1_MAX_RX_QUEUES)

typedef
#include "vmware_pack_begin.h"
struct UPT1_TxStats {
   uint64 TSOPktsTxOK;  /* TSO pkts post-segmentation */
   uint64 TSOBytesTxOK;
   uint64 ucastPktsTxOK;
   uint64 ucastBytesTxOK;
   uint64 mcastPktsTxOK;
   uint64 mcastBytesTxOK;
   uint64 bcastPktsTxOK;
   uint64 bcastBytesTxOK;
   uint64 pktsTxError;
   uint64 pktsTxDiscard;
}
#include "vmware_pack_end.h"
UPT1_TxStats;

typedef
#include "vmware_pack_begin.h"
struct UPT1_RxStats {
   uint64 LROPktsRxOK;    /* LRO pkts */
   uint64 LROBytesRxOK;   /* bytes from LRO pkts */
   /* the following counters are for pkts from the wire, i.e., pre-LRO */
   uint64 ucastPktsRxOK;
   uint64 ucastBytesRxOK;
   uint64 mcastPktsRxOK;
   uint64 mcastBytesRxOK;
   uint64 bcastPktsRxOK;
   uint64 bcastBytesRxOK;
   uint64 pktsRxOutOfBuf;
   uint64 pktsRxError;
}
#include "vmware_pack_end.h"
UPT1_RxStats;

/* interrupt moderation level */
#define UPT1_IML_NONE     0 /* no interrupt moderation */
#define UPT1_IML_HIGHEST  7 /* least intr generated */
#define UPT1_IML_ADAPTIVE 8 /* adpative intr moderation */

/* values for UPT1_RSSConf.hashFunc */
#define UPT1_RSS_HASH_TYPE_NONE      0x0
#define UPT1_RSS_HASH_TYPE_IPV4      0x01
#define UPT1_RSS_HASH_TYPE_TCP_IPV4  0x02
#define UPT1_RSS_HASH_TYPE_IPV6      0x04
#define UPT1_RSS_HASH_TYPE_TCP_IPV6  0x08

#define UPT1_RSS_HASH_FUNC_NONE      0x0
#define UPT1_RSS_HASH_FUNC_TOEPLITZ  0x01

#define UPT1_RSS_MAX_KEY_SIZE        40
#define UPT1_RSS_MAX_IND_TABLE_SIZE  128

typedef 
#include "vmware_pack_begin.h"
struct UPT1_RSSConf {
   uint16   hashType;
   uint16   hashFunc;
   uint16   hashKeySize;
   uint16   indTableSize;
   uint8    hashKey[UPT1_RSS_MAX_KEY_SIZE];
   uint8    indTable[UPT1_RSS_MAX_IND_TABLE_SIZE];
}
#include "vmware_pack_end.h"
UPT1_RSSConf;

/* features */
#define UPT1_F_RXCSUM      0x0001   /* rx csum verification */
#define UPT1_F_RSS         0x0002
#define UPT1_F_RXVLAN      0x0004   /* VLAN tag stripping */
#define UPT1_F_LRO         0x0008
#define UPT1_F_INNER_LRO   0x0010   /* LRO for Geneve/VXLAN encap packets */

#endif
�������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������